diff --git a/runtime/include/guest_abi/dispatch.h b/runtime/include/guest_abi/dispatch.h index dba9ef15e..d038033e6 100644 --- a/runtime/include/guest_abi/dispatch.h +++ b/runtime/include/guest_abi/dispatch.h @@ -130,7 +130,6 @@ inline const RawDispatchRecord* FindStaticIndirectDispatchEntry( } void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* table); -inline std::atomic g_publishedStaticIndirectDispatchTable{nullptr}; class StaticIndirectDispatchTableRegistrar { public: @@ -298,8 +297,8 @@ struct IsPureTypedNativeCpuCall> : std::bool_ #include "guest_abi/native_cpu_calls.inc" -// Direct-mapped thread-local memo in front of the sorted registry lookup every bctrl performs; -// a miss just re-runs the sorted lookup. 512 entries (12 KiB) covers the per-frame indirect +// Direct-mapped thread-local memo in front of the registry lookup for targets without a raw +// dispatch record; a miss just re-runs the lookup. 512 entries (12 KiB) covers the per-frame indirect // working set while staying L1/L2 resident, unlike 4096 which would thrash L2. Must stay a // power of two, the index masks with (size - 1). inline constexpr size_t kIndirectDispatchCacheEntries = 512; @@ -314,13 +313,23 @@ struct IndirectResolvedDispatchMemoEntry { inline thread_local IndirectResolvedDispatchMemoEntry g_indirectResolvedDispatchMemo[kIndirectDispatchCacheEntries]{}; -struct IndirectRawDispatchMemoEntry { - bool valid; +// Every raw dispatch record in one open-addressed table, built by Finalize and shared by all +// threads. Kept at most half full, so a lookup almost always reads a single slot, and unlike a +// per-thread memo it never misses into the generated table's page search. +struct RawDispatchHashSlot { uint32_t address; - const RawDispatchRecord* record; + const RawDispatchRecord* record; // nullptr = empty +}; + +struct RawDispatchHashTable { + const RawDispatchHashSlot* slots; + uint32_t mask; // slot count - 1 + uint32_t shift; // 32 - log2(slot count); Fibonacci hashing keeps the product's top bits }; -inline thread_local IndirectRawDispatchMemoEntry - g_indirectRawDispatchMemo[kIndirectDispatchCacheEntries]{}; + +MKW_PPC_FORCE_INLINE uint32_t RawDispatchHashIndex(const RawDispatchHashTable& table, uint32_t address) { + return ((address >> 2) * 2654435761u) >> table.shift; +} class TranslatedFunctionRegistry { public: @@ -345,48 +354,29 @@ class TranslatedFunctionRegistry { return info; } - // Hot wrapper: only the direct-mapped memo probe stays inline. Inlining the - // generated-table binary search alongside it pushed the whole function past - // the inliner's budget and out-of-lined the probe itself, which is the part - // every bctrl executes. + // nullptr before Finalize, like the publication flag this replaced. static MKW_PPC_FORCE_INLINE const RawDispatchRecord* FindRawByAddressPtr(uint32_t address) { - if (!lookupPublished_.load(std::memory_order_acquire)) { + const RawDispatchHashTable* table = rawDispatchTable_.load(std::memory_order_acquire); + if (table == nullptr) { return nullptr; } - - auto& cached = g_indirectRawDispatchMemo[ - ((address >> 2) * 2654435761u) & (kIndirectDispatchCacheEntries - 1)]; - if (cached.valid && cached.address == address) { - return cached.record; + for (uint32_t index = RawDispatchHashIndex(*table, address);; index = (index + 1) & table->mask) { + const RawDispatchHashSlot& slot = table->slots[index]; + if (slot.record == nullptr || slot.address == address) { + return slot.record; + } } - - return FindRawByAddressPtrMiss(address, cached); } private: - // Memo miss: resolve against the header-inline generated table first; only dynamically - // registered records or a not-yet-published table fall through to the slow path. - static MKW_PPC_NO_INLINE MKW_PPC_COLD const RawDispatchRecord* FindRawByAddressPtrMiss( - uint32_t address, IndirectRawDispatchMemoEntry& cached) { - const RawDispatchRecord* record = nullptr; - if (const auto* table = - g_publishedStaticIndirectDispatchTable.load(std::memory_order_acquire)) { - record = FindStaticIndirectDispatchEntry(table, address); - } - if (record == nullptr) { - record = FindRawByAddressPtrSlow(address); - } - cached = {true, address, record}; - return record; - } - static const TranslatedFunctionInfo* FindByAddressPtrSlow(uint32_t address); - static const RawDispatchRecord* FindRawByAddressPtrSlow(uint32_t address); + static void BuildRawDispatchTableLocked(); // Finalize publishes immutable registry, generated-table, and dynamic raw // dispatch storage. Header-inlined cache hits are enabled only after that // release publication, so a pre-finalization miss can never poison them. inline static std::atomic_bool lookupPublished_{false}; + inline static std::atomic rawDispatchTable_{nullptr}; }; // Cold, profile-owned registration data is emitted in compact table shards. diff --git a/runtime/include/isa/ppc_isa_float.h b/runtime/include/isa/ppc_isa_float.h index 4aa6f4eb1..c01377145 100644 --- a/runtime/include/isa/ppc_isa_float.h +++ b/runtime/include/isa/ppc_isa_float.h @@ -191,7 +191,7 @@ inline bool MkwHostNiActiveInline() noexcept return g_mkwHostNiActive; } -inline float PpcForceSingleValueInline(double value) +inline float PpcForceSingleValueSlowInline(double value) { // FPSCR[NI] flushes an exact pre-round single-subnormal even when rounding would promote it // to the smallest normal. g_mkwNiFlushThreshold (2^-126 active, 0.0 inactive) turns the @@ -216,6 +216,18 @@ inline float PpcForceSingleValueInline(double value) #endif } +inline float PpcForceSingleValueInline(double value) +{ + // Convert first and take the exact pre-round check above only for a +-FLT_MIN or subnormal + // result. A larger magnitude means |value| was at least 2^-126 (no rounding mode carries a + // smaller one past FLT_MIN), +-0 is what the flush produces anyway, and NaN/infinity never + // flush, so every other result matches. This keeps the mask off each single op's dependency chain. + const float converted = static_cast(value); + if ((PpcBitCastToU32Inline(converted) & 0x7FFFFFFFu) - 1u < 0x00800000u) [[unlikely]] + return PpcForceSingleValueSlowInline(value); + return converted; +} + inline float PpcFlushSingleForNiInline(float value) { if (!MkwHostNiActiveInline()) diff --git a/runtime/src/abi_bridge.cpp b/runtime/src/abi_bridge.cpp index 55fe41651..6c7c67cd6 100644 --- a/runtime/src/abi_bridge.cpp +++ b/runtime/src/abi_bridge.cpp @@ -307,9 +307,6 @@ void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* tabl std::abort(); } registered = table; - // Mirror into the header-visible atomic under the same lock, so the - // inlined miss path and this owning static can never disagree. - g_publishedStaticIndirectDispatchTable.store(table, std::memory_order_release); } void RegisterBulkTranslatedFunctions(const BulkTranslatedFunctionRecord* records, size_t count) { @@ -398,6 +395,7 @@ void TranslatedFunctionRegistry::Finalize() { auto& addressEntries = AddressEntries(); addressEntries.clear(); addressEntries.rehash(0); + BuildRawDispatchTableLocked(); RegistryFrozen().store(true, std::memory_order_release); lookupPublished_.store(true, std::memory_order_release); } @@ -478,19 +476,39 @@ GuestFunctions::Entry GuestFunctions::NextTranslation(uint32_t address) { ReportMissingGuestFunction(address); } -const RawDispatchRecord* TranslatedFunctionRegistry::FindRawByAddressPtrSlow(uint32_t address) { - if (!RegistryFrozen().load(std::memory_order_acquire)) { - return nullptr; +void TranslatedFunctionRegistry::BuildRawDispatchTableLocked() { + const auto* generated = GeneratedIndirectDispatchTable(); + const auto& dynamic = DynamicRawDispatchEntries(); + const size_t generatedCount = generated != nullptr ? generated->entryCount : 0; + uint32_t bits = 10; + while ((size_t{1} << bits) < (generatedCount + dynamic.size()) * 2) { + ++bits; + } + static std::vector slots; + static RawDispatchHashTable table; + slots.assign(size_t{1} << bits, RawDispatchHashSlot{0, nullptr}); + table = {slots.data(), (1u << bits) - 1u, 32u - bits}; + // The first record put at an address wins: the generated table, then the dynamic records in + // address order, the precedence the sorted lookups gave. + const auto put = [](const RawDispatchRecord& record) { + for (uint32_t index = RawDispatchHashIndex(table, record.address);; index = (index + 1) & table.mask) { + RawDispatchHashSlot& slot = slots[index]; + if (slot.record == nullptr) { + slot = {record.address, &record}; + return; + } + if (slot.address == record.address) { + return; + } + } + }; + for (size_t i = 0; i < generatedCount; ++i) { + put(generated->entries[i]); } - if (const auto* generated = FindStaticIndirectDispatchEntry(GeneratedIndirectDispatchTable(), address)) { - return generated; + for (const auto& record : dynamic) { + put(record); } - const auto& entries = DynamicRawDispatchEntries(); - const auto it = std::lower_bound(entries.begin(), entries.end(), address, - [](const RawDispatchRecord& entry, uint32_t target) { - return entry.address < target; - }); - return it != entries.end() && it->address == address ? &*it : nullptr; + rawDispatchTable_.store(&table, std::memory_order_release); } std::optional FindTranslatedFunctionByHostAddress(uintptr_t hostAddr) { diff --git a/runtime/src/hle/gx/gx_copy.cpp b/runtime/src/hle/gx/gx_copy.cpp index 3dbdc8d6e..2f13b3b50 100644 --- a/runtime/src/hle/gx/gx_copy.cpp +++ b/runtime/src/hle/gx/gx_copy.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace { @@ -20,12 +21,47 @@ std::mutex g_efbCopyDestinationsMutex; std::map g_efbCopyDestinations; uint32_t g_largestEfbCopyDestination = 0; +// How many destinations touch each 64 KiB bucket of guest RAM. Nearly every DCStoreRange misses all +// destinations, and an empty bucket answers that without walking the map. +constexpr uint32_t kEfbCopyBucketShift = 16; +std::unordered_map g_efbCopyBuckets; + +void CountEfbCopyBucketsLocked(uint64_t start, uint32_t size, int32_t delta) { + const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift; + for (uint64_t bucket = start >> kEfbCopyBucketShift; bucket <= last; ++bucket) { + g_efbCopyBuckets[static_cast(bucket)] += static_cast(delta); + } +} + +bool AnyEfbCopyBucketLocked(uint64_t start, uint32_t size) { + const uint64_t first = start >> kEfbCopyBucketShift; + const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift; + if (last - first + 1u > g_efbCopyBuckets.size()) { + return true; // wider than the index; the map walk answers + } + for (uint64_t bucket = first; bucket <= last; ++bucket) { + const auto it = g_efbCopyBuckets.find(static_cast(bucket)); + if (it != g_efbCopyBuckets.end() && it->second != 0) { + return true; + } + } + return false; +} + void RememberEfbCopyDestination(uint32_t addr, uint32_t size) { if (size == 0) { return; } std::lock_guard guard(g_efbCopyDestinationsMutex); - g_efbCopyDestinations[CanonicalizeGxMainRamAddress(addr)] = size; + const uint32_t canonical = CanonicalizeGxMainRamAddress(addr); + const auto [it, inserted] = g_efbCopyDestinations.try_emplace(canonical, size); + if (inserted) { + CountEfbCopyBucketsLocked(canonical, size, +1); + } else if (it->second != size) { + CountEfbCopyBucketsLocked(canonical, it->second, -1); + CountEfbCopyBucketsLocked(canonical, size, +1); + it->second = size; + } // A conservative monotonic maximum lets invalidation jump directly to // the only map interval that could overlap instead of walking every copy // destination on every DCStoreRange. @@ -44,6 +80,9 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) { std::vector retired; { std::lock_guard guard(g_efbCopyDestinationsMutex); + if (!AnyEfbCopyBucketLocked(dirtyStart, size)) { + return; + } const uint64_t earliestCandidate = dirtyStart > g_largestEfbCopyDestination ? dirtyStart - g_largestEfbCopyDestination : 0; for (auto it = g_efbCopyDestinations.lower_bound(static_cast(earliestCandidate)); @@ -55,6 +94,7 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) { continue; } retired.push_back(it->first); + CountEfbCopyBucketsLocked(it->first, it->second, -1); it = g_efbCopyDestinations.erase(it); } } diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index daceeb785..71e18fbcd 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1207,6 +1207,27 @@ static void SubmitLytDrawPacket(const uint8_t* packet, uint32_t packetBytes) { SyncAppliedVtxStateFromHleReal(); } +// A span inside one fully mapped 1 MiB page resolves straight from the page bias, the pointer +// Contains + GuestToHostPtr would give after two table lookups. +static const uint8_t* ResolveDisplayListArray(uint32_t base, uint32_t span) { +#if !defined(__APPLE__) + // macOS has to keep GuestToHostPtr's canonical alias, which is the GX resource identity there. + if (base != 0 && span != 0) { + const uint32_t page = base >> MemoryInline::kPageShift; + const uint32_t last = base + span - 1u; + if (last >= base && (last >> MemoryInline::kPageShift) == page) { + if (const uintptr_t bias = MemoryInline::g_fullPageBias[page]; bias != 0) { + return reinterpret_cast((bias - 1u) + base); + } + } + } +#endif + if (!Memory::Contains(base, span)) { + return nullptr; + } + return static_cast(GuestToHostPtr(base, span)); +} + static bool ApplyAuroraArraysForDisplayList(const std::array& maxIdx, const std::array& sawIdx, GXVtxFmt vtxfmt, @@ -1232,11 +1253,7 @@ static bool ApplyAuroraArraysForDisplayList(const std::array 0) ? (static_cast((count - 1u) * arr.stride) + elemSize) : 0u; const bool needsPack = arr.stride > 0xffu; - if (!Memory::Contains(arr.base, span)) { - ok = false; - continue; - } - const uint8_t* hostPtr = static_cast(GuestToHostPtr(arr.base, span)); + const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span); if (!hostPtr) { ok = false; continue; @@ -1273,11 +1290,7 @@ static bool ApplyAuroraIndexedXFArraysForDisplayList(const std::array(GuestToHostPtr(arr.base, span)); + const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span); if (!hostPtr) { ok = false; continue; diff --git a/runtime/src/hle/gx/gx_fifo.cpp b/runtime/src/hle/gx/gx_fifo.cpp index 108e4fd55..f122bd292 100644 --- a/runtime/src/hle/gx/gx_fifo.cpp +++ b/runtime/src/hle/gx/gx_fifo.cpp @@ -343,9 +343,39 @@ void SubmitAttribute(GXAttr attr, float* comps, const VtxAttrFmt& fmt, const u32 } } +// A word that lands inside an XF load still short of its last word only appends: the general +// path below would push it and then stop parsing at the same incomplete packet. +static bool TryAppendIncompleteXfWord(u32 value) { + auto& g = g_hleGxState; + if (g.inBegin || g.fifoByteCount < 5u || + g.fifoReadOffset + g.fifoByteCount + 4u > g.fifoBytes.size()) { + return false; + } + uint8_t* data = g.fifoBytes.data() + g.fifoReadOffset; + if ((data[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) { + return false; + } + const uint32_t packetBytes = 5u + (static_cast(ReadBE16(data + 1)) + 1u) * 4u; + if (g.fifoByteCount + 4u >= packetBytes) { + return false; + } + BigEndian::Write32(data + g.fifoByteCount, value); + g.fifoByteCount += 4u; + return true; +} + +static MKW_PPC_NO_INLINE void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes); + // `val` is a raw big-endian bit pattern: the FIFO stream is type-agnostic, and // the float entry point converts before it gets here. void HleFifoWrite(u32 val, uint32_t sizeBytes) { + if ((sizeBytes == 0 || sizeBytes == 4) && !IsDisplayListActive() && TryAppendIncompleteXfWord(val)) { + return; + } + HleFifoWriteGeneral(val, sizeBytes); +} + +static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { const bool recordOnly = IsDisplayListActive(); if (recordOnly) { WriteDisplayListData(val, sizeBytes); @@ -727,9 +757,19 @@ static uint32_t ApplyFifoPacketsDirect(const uint8_t* data, uint32_t sizeBytes) const uint16_t countWords = ReadBE16(packet + 1); const uint32_t packetBytes = 1u + 4u + (static_cast(countWords) + 1u) * 4u; if (avail < packetBytes) break; - GXCallDisplayList(packet, packetBytes); + // Aurora parses a run of adjacent XF loads in one call. The run stays inside this + // burst, so it keeps its order against the CP/BP writes around it. + uint32_t runBytes = packetBytes; + while (avail - runBytes >= 5u) { + const uint8_t* next = packet + runBytes; + if ((next[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) break; + const uint32_t nextBytes = 5u + (static_cast(ReadBE16(next + 1)) + 1u) * 4u; + if (avail - runBytes < nextBytes) break; + runBytes += nextBytes; + } + GXCallDisplayList(packet, runBytes); GXMarkFrameWork(); - offset += packetBytes; + offset += runBytes; continue; } diff --git a/runtime/src/hle/storage/dvd.cpp b/runtime/src/hle/storage/dvd.cpp index b76e5e19c..2c55c46eb 100644 --- a/runtime/src/hle/storage/dvd.cpp +++ b/runtime/src/hle/storage/dvd.cpp @@ -103,10 +103,23 @@ extern "C" void DVDInit_8015EA1C(); extern "C" uint32_t g_dvdFstReservedBase; extern "C" uint32_t g_dvdFstReservedSize; -// Byte-wise copy into guest RAM plus the DMA notification the GX caches need. +// Copy into guest RAM plus the DMA notification the GX caches need. A fully writable 1 MiB page +// takes its bytes in one memcpy, since Write8 is a plain store there; any other page keeps Write8. static void CopyToGuestAsDma(uint32_t dest, const uint8_t* data, size_t size) { - for (size_t i = 0; i < size; ++i) { - Memory::Write8(dest + static_cast(i), data[i]); + size_t done = 0; + while (done < size) { + const uint32_t address = dest + static_cast(done); + const size_t pageLeft = MemoryInline::kPageSize - (address & MemoryInline::kPageMask); + const size_t chunk = std::min(size - done, pageLeft); + if (const uintptr_t bias = MemoryInline::g_fullWritablePageBias[address >> MemoryInline::kPageShift]; + bias != 0) { + std::memcpy(reinterpret_cast((bias - 1u) + address), data + done, chunk); + } else { + for (size_t i = 0; i < chunk; ++i) { + Memory::Write8(address + static_cast(i), data[done + i]); + } + } + done += chunk; } GxNotifyGuestRamDmaWrite(dest, static_cast(size)); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs index c5d3cd95f..a561e0a89 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs @@ -560,7 +560,7 @@ _activeResidency is not null && // Tail dispatch: the callee inherits authoritative context // and this frame never observes registers again, so the // locals must not be written back afterwards. - AppendFlush(sb, pad); + AppendExitFlush(sb, pad); sb.AppendLine($"{pad}InvokeIndirectJump({jumpTarget}, ctx);"); sb.AppendLine($"{pad}return;"); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs index 9c03dbacc..a7bd0cbd0 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs @@ -26,13 +26,105 @@ private sealed class RegisterResidency private bool _writesCtr; private bool _writesXer; - private RegisterResidency(bool recording) => _recording = recording; + // Dirty-register tracking (CpuContext write-back only for registers whose local changed since the last sync). + // A register is dirty while its local may differ from CpuContext: set by a write, cleared by an entry load, + // a reload after a boundary, or a write-back. The recording pass summarizes every block as a gen/kill + // transfer plus its outgoing edge writes; Snapshot() solves the forward union dataflow over them; the + // real pass replays the same transfer from the solved block-entry state at every flush point. + // A function that can resume through a local LR-continuation label has edges this analysis does not + // see, so it keeps the old write-back-everything-written behaviour. + private readonly string? _entryLabel; + private readonly bool _conservative; + private Dictionary? _blocks; + private Dictionary? _blockIn; + private BlockSummary? _activeBlock; + private DirtySet _activeIn; + private string? _edgeTarget; + private DirtySet _edgeGen; + private readonly Stack<(Transfer Base, List Done)> _alternatives = new(); + + private const byte CrBit = 1; + private const byte CtrBit = 2; + private const byte XerBit = 4; + + // What a guest callback may leave changed in CpuContext under the PowerPC EABI (r0, r3-r12, + // f0-f13, CR, CTR, XER), plus r13, which OS_HLE_ProcessAlarms resets to the SDA base. A runtime + // native can run guest handlers on the caller's live context behind a contract that declares no + // writes (OS_HLE_ProcessAlarms from GKI_delay / PPCMtdec), so after any boundary these stay dirty + // unless it reloaded them; the callbacks preserve the rest. + private static readonly DirtySet CallbackClobberable = new() + { + Gpr = 0x00003FF9u, + Fpr = 0x00003FFFu, + Flags = CrBit | CtrBit | XerBit + }; + + internal struct DirtySet : IEquatable + { + public uint Gpr; + public uint Fpr; + public byte Flags; + + public static DirtySet Or(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr | b.Gpr, Fpr = a.Fpr | b.Fpr, Flags = (byte)(a.Flags | b.Flags) }; + + public static DirtySet And(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr & b.Gpr, Fpr = a.Fpr & b.Fpr, Flags = (byte)(a.Flags & b.Flags) }; + + public static DirtySet AndNot(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr & ~b.Gpr, Fpr = a.Fpr & ~b.Fpr, Flags = (byte)(a.Flags & ~b.Flags) }; + + public bool Equals(DirtySet other) => Gpr == other.Gpr && Fpr == other.Fpr && Flags == other.Flags; + public override bool Equals(object? obj) => obj is DirtySet other && Equals(other); + public override int GetHashCode() => HashCode.Combine(Gpr, Fpr, Flags); + } + + /// out = (in minus Kill) plus Gen. Closed under sequencing and under join of alternatives. + internal struct Transfer + { + public DirtySet Gen; + public DirtySet Kill; - public static RegisterResidency CreateRecorder() => new(true); + public readonly DirtySet Apply(DirtySet input) => DirtySet.Or(DirtySet.AndNot(input, Kill), Gen); + + public void Write(DirtySet written) + { + Gen = DirtySet.Or(Gen, written); + Kill = DirtySet.AndNot(Kill, written); + } + + public void Clean(DirtySet cleaned) + { + Gen = DirtySet.AndNot(Gen, cleaned); + Kill = DirtySet.Or(Kill, cleaned); + } + + public static Transfer Join(Transfer a, Transfer b) => new() + { + Gen = DirtySet.Or(a.Gen, b.Gen), + Kill = DirtySet.And(a.Kill, b.Kill) + }; + } + + private sealed class BlockSummary + { + public Transfer Transfer; + public readonly List<(string Target, DirtySet Gen)> Edges = new(); + } + + private RegisterResidency(bool recording, string? entryLabel = null, bool conservative = false) + { + _recording = recording; + _entryLabel = entryLabel; + _conservative = conservative; + } + + public static RegisterResidency CreateRecorder(string? entryLabel = null, bool conservative = false) => + new(true, entryLabel, conservative) { _blocks = new Dictionary(StringComparer.OrdinalIgnoreCase) }; public RegisterResidency Snapshot() { - var snapshot = new RegisterResidency(false); + var snapshot = new RegisterResidency(false, _entryLabel, _conservative); snapshot._gprs.UnionWith(_gprs); snapshot._fprs.UnionWith(_fprs); snapshot._writtenGprs.UnionWith(_writtenGprs); @@ -43,9 +135,143 @@ public RegisterResidency Snapshot() snapshot._writesCr = _writesCr; snapshot._writesCtr = _writesCtr; snapshot._writesXer = _writesXer; + snapshot._blocks = _blocks; + snapshot._blockIn = SolveDirtyEntryStates(snapshot.WrittenSet()); return snapshot; } + private DirtySet WrittenSet() + { + var set = new DirtySet(); + foreach (var gpr in _writtenGprs) set.Gpr |= 1u << gpr; + foreach (var fpr in _writtenFprs) set.Fpr |= 1u << fpr; + if (_writesCr) set.Flags |= CrBit; + if (_writesCtr) set.Flags |= CtrBit; + if (_writesXer) set.Flags |= XerBit; + return set; + } + + /// + /// Forward union dataflow over the recorded blocks. The entry block starts clean (entry loads); a block no + /// recorded edge reaches (and that is not the entry) starts with everything dirty, the old behaviour. + /// + private Dictionary? SolveDirtyEntryStates(DirtySet all) + { + if (_blocks is null || _conservative || _entryLabel is null || !_blocks.ContainsKey(_entryLabel)) + return null; + + var ins = new Dictionary(StringComparer.OrdinalIgnoreCase); + var reached = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var block in _blocks.Values) + foreach (var edge in block.Edges) + reached.Add(edge.Target); + foreach (var label in _blocks.Keys) + ins[label] = !string.Equals(label, _entryLabel, StringComparison.OrdinalIgnoreCase) && !reached.Contains(label) + ? all + : default; + + bool changed; + do + { + changed = false; + foreach (var (label, block) in _blocks) + { + var output = block.Transfer.Apply(ins[label]); + foreach (var (target, edgeGen) in block.Edges) + { + if (!ins.TryGetValue(target, out var current)) continue; + var merged = DirtySet.Or(current, DirtySet.Or(output, edgeGen)); + if (!merged.Equals(current)) + { + ins[target] = merged; + changed = true; + } + } + } + } + while (changed); + + return ins; + } + + /// Starts a block: the running transfer is empty and, in the real pass, its entry state is the solved one. + public void BeginBlock(string label) + { + _activeBlock = new BlockSummary(); + _edgeTarget = null; + _alternatives.Clear(); + if (_recording) + { + _blocks![label] = _activeBlock; + _activeIn = default; + } + else + { + _activeIn = _blockIn is not null && _blockIn.TryGetValue(label, out var known) ? known : WrittenSet(); + } + } + + /// Writes made while a CFG edge's normalization is emitted belong to that edge only. + public void BeginEdge(string target) + { + if (_activeBlock is null) return; + _edgeTarget = target; + _edgeGen = default; + } + + public void EndEdge() + { + if (_activeBlock is null || _edgeTarget is null) return; + if (_recording) _activeBlock.Edges.Add((_edgeTarget, _edgeGen)); + _edgeTarget = null; + } + + /// + /// Brackets code whose arms are alternatives within one block (a state-free call's fast and slow arm): + /// each arm starts from the state at ; afterwards the state is their join. + /// + public void BeginAlternatives() + { + if (_activeBlock is null) return; + _alternatives.Push((_activeBlock.Transfer, new List())); + } + + public void NextAlternative() + { + if (_activeBlock is null) return; + var (baseTransfer, done) = _alternatives.Peek(); + done.Add(_activeBlock.Transfer); + _activeBlock.Transfer = baseTransfer; + } + + public void EndAlternatives() + { + if (_activeBlock is null) return; + var (_, done) = _alternatives.Pop(); + var joined = _activeBlock.Transfer; + foreach (var arm in done) joined = Transfer.Join(joined, arm); + _activeBlock.Transfer = joined; + } + + private DirtySet CurrentDirty() + { + if (_activeBlock is null || _conservative) return WrittenSet(); + return _activeBlock.Transfer.Apply(_activeIn); + } + + private void NoteWrite(DirtySet written) + { + if (_activeBlock is null) return; + if (_edgeTarget is not null) _edgeGen = DirtySet.Or(_edgeGen, written); + else _activeBlock.Transfer.Write(written); + } + + private void NoteClean(DirtySet cleaned) + { + if (_activeBlock is null || _edgeTarget is not null) return; + _activeBlock.Transfer.Clean(cleaned); + } + public bool IsEmpty => _gprs.Count == 0 && _fprs.Count == 0 && !_cr && !_ctr && !_xer; public static string GprLocal(int index) => GprRegisterName(index); @@ -88,6 +314,7 @@ public string FprStorage(int index, bool written) public string Cr(bool written) { + if (written) NoteWrite(new DirtySet { Flags = CrBit }); if (_recording) { _cr = true; @@ -104,6 +331,7 @@ public string Cr(bool written) public string Ctr(bool written) { + if (written) NoteWrite(new DirtySet { Flags = CtrBit }); if (_recording) { _ctr = true; @@ -124,6 +352,7 @@ public string Ctr(bool written) /// public string Xer(bool written) { + if (written) NoteWrite(new DirtySet { Flags = XerBit }); if (_recording) { _xer = true; @@ -140,6 +369,12 @@ public string Xer(bool written) private void Track(SortedSet referenced, SortedSet written, int index, bool isWrite, string prefix) { + if (isWrite) + { + var bit = 1u << index; + NoteWrite(ReferenceEquals(referenced, _gprs) ? new DirtySet { Gpr = bit } : new DirtySet { Fpr = bit }); + } + if (_recording) { referenced.Add(index); @@ -183,18 +418,36 @@ public void AppendEntryLoads(StringBuilder sb) /// or overwrite are published; see /// for why the write set has to participate in the flush mask. /// - public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync) + public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync) => + AppendFlush(sb, pad, sync, leavesFrame: false); + + /// + /// Writes back the registers that are both dirty and in the boundary's flush set. A flush that stays in + /// this frame ( false) leaves them clean afterwards; one that returns or + /// tail-dispatches must not change the state the sibling paths of the block see. + /// + public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync, bool leavesFrame) { - if (_recording) return; - foreach (var gpr in _writtenGprs) - if (sync.FlushesGpr(gpr)) - sb.Append(pad).Append("ctx->gpr[").Append(gpr).Append("] = ").Append(GprLocal(gpr)).AppendLine(";"); - foreach (var fpr in _writtenFprs) - if (sync.FlushesFpr(fpr)) - sb.Append(pad).Append("ctx->fpr[").Append(fpr).Append("] = ").Append(FprLocal(fpr)).AppendLine(";"); - if (_writesCr && sync.FlushesCr) sb.Append(pad).AppendLine($"ctx->cr = {CrLocal};"); - if (_writesCtr && sync.FlushesCtr) sb.Append(pad).AppendLine($"ctx->ctr = {CtrLocal};"); - if (_writesXer && sync.FlushesXer) sb.Append(pad).AppendLine($"ctx->xer = {XerLocal};"); + var dirty = CurrentDirty(); + if (!_recording) + { + foreach (var gpr in _writtenGprs) + if (sync.FlushesGpr(gpr) && (dirty.Gpr & (1u << gpr)) != 0) + sb.Append(pad).Append("ctx->gpr[").Append(gpr).Append("] = ").Append(GprLocal(gpr)).AppendLine(";"); + foreach (var fpr in _writtenFprs) + if (sync.FlushesFpr(fpr) && (dirty.Fpr & (1u << fpr)) != 0) + sb.Append(pad).Append("ctx->fpr[").Append(fpr).Append("] = ").Append(FprLocal(fpr)).AppendLine(";"); + if (_writesCr && sync.FlushesCr && (dirty.Flags & CrBit) != 0) sb.Append(pad).AppendLine($"ctx->cr = {CrLocal};"); + if (_writesCtr && sync.FlushesCtr && (dirty.Flags & CtrBit) != 0) sb.Append(pad).AppendLine($"ctx->ctr = {CtrLocal};"); + if (_writesXer && sync.FlushesXer && (dirty.Flags & XerBit) != 0) sb.Append(pad).AppendLine($"ctx->xer = {XerLocal};"); + } + + if (leavesFrame) return; + var flushed = new DirtySet { Gpr = sync.FlushGprMask, Fpr = sync.FlushFprMask }; + if (sync.FlushesCr) flushed.Flags |= CrBit; + if (sync.FlushesCtr) flushed.Flags |= CtrBit; + if (sync.FlushesXer) flushed.Flags |= XerBit; + NoteClean(flushed); } /// @@ -206,6 +459,12 @@ public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync /// Contract-narrowed reload. public void AppendReload(StringBuilder sb, string pad, ResidencyBoundarySync sync) { + var reloaded = new DirtySet { Gpr = sync.ReloadGprMask, Fpr = sync.ReloadFprMask }; + if (sync.ReloadsCr) reloaded.Flags |= CrBit; + if (sync.ReloadsCtr) reloaded.Flags |= CtrBit; + if (sync.ReloadsXer) reloaded.Flags |= XerBit; + NoteWrite(DirtySet.AndNot(CallbackClobberable, reloaded)); + NoteClean(reloaded); if (_recording) return; foreach (var gpr in _gprs) if (sync.ReloadsGpr(gpr)) @@ -348,6 +607,12 @@ private static void AppendFlush(StringBuilder sb, string pad, ResidencyBoundaryS if (_activeResidency is { } residency) residency.AppendFlush(sb, pad, sync); } + /// Write-back on a path that leaves the frame (return, tail dispatch): publishes the dirty registers, tracks nothing. + private static void AppendExitFlush(StringBuilder sb, string pad) + { + if (_activeResidency is { } residency) residency.AppendFlush(sb, pad, ResidencyBoundarySync.Full, leavesFrame: true); + } + private static void AppendReload(StringBuilder sb, string pad, ResidencyBoundarySync sync) { if (_activeResidency is { } residency) residency.AppendReload(sb, pad, sync); diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs index 224b30334..f8adb3050 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs @@ -375,6 +375,9 @@ private static void EmitResidentStateFreeCall( $"Resident state-free marshalling requested for 0x{address:X8} without an active residency."); AppendFlush(sb, pad, ResidencyBoundarySync.StateFreeResidentOuter); + // The fast arm's result stores and the slow arm's flush/reload are alternatives: each starts from + // the state here and the dirty set afterwards is their join. + residency.BeginAlternatives(); var inner = pad + " "; sb.AppendLine($"{pad}if ({StateFreeAvailabilityCondition(address, stateFreeAbiContracts, stateFreeCallSymbols)}) {{"); var arguments = StateFreeResidentCallArguments(residency, contract, callSiteLr); @@ -391,10 +394,12 @@ private static void EmitResidentStateFreeCall( sb.AppendLine($"{inner}{symbol}({arguments});"); } sb.AppendLine($"{pad}}} else {{"); + residency.NextAlternative(); AppendFlush(sb, inner, ResidencyBoundarySync.Full); RecordEmittedDirectCallTarget(address); sb.AppendLine($"{inner}InvokeDirectCpu<0x{address:X8}u>(ctx);"); AppendReload(sb, inner, ResidencyBoundarySync.Full); + residency.EndAlternatives(); sb.AppendLine($"{pad}}}"); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs index 2e0cee994..019d8dca6 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs @@ -434,6 +434,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool var localConstants = new Dictionary(StringComparer.OrdinalIgnoreCase); var label = labelNames[block.Label]; RecordEmittedLocalLabel(label); + _activeResidency?.BeginBlock(block.Label); body.AppendLine($"{label}:"); body.AppendLine("{"); var localPaired = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -523,12 +524,12 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool { case IrUndefined undef: EmitInstruction(block.Label, -1, undef, body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase: false, localFallthroughLr: null, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch, resumeLabelPairedStates); - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine(" return;"); break; case IrReturn ret: - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine(" return;"); break; @@ -576,7 +577,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool // Unlisted selectors leave this frame through the // dispatcher: publish the locals and never write // them back afterwards. - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine($" InvokeIndirectJump({selectorExpr}, ctx);"); body.AppendLine(" return;"); body.AppendLine(" }"); @@ -597,7 +598,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool // unreachable and must not write them back. if (term is not IrIndirectJump) { - AppendFlush(body, " "); + AppendExitFlush(body, " "); } body.AppendLine(" return;"); } @@ -647,7 +648,9 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool Environment.NewLine.Length; RegisterResidency? residency = null; { - var recorder = RegisterResidency.CreateRecorder(); + // A body that can resume through a local LR-continuation label has CFG edges the dirty-register + // analysis does not model, so it keeps the write-back-everything-written behaviour. + var recorder = RegisterResidency.CreateRecorder(func.EntryLabel, conservative: continuationCallCount > 0); EmitFunctionBody(recorder, bodyBufferBaseLength, discardResult: true); var discovered = recorder.Snapshot(); if (!discovered.IsEmpty) @@ -1170,6 +1173,9 @@ private static void EmitNormalizePairedStateOnEdge( IReadOnlyDictionary> pairedIn, IReadOnlyDictionary>? liveIn) { + // The edge's own register writes (normalization) and its target are what the dirty-register + // dataflow needs; every emitted CFG edge goes through here. + _activeResidency?.BeginEdge(toLabel); pairedOut.TryGetValue(fromLabel, out var fromSet); pairedIn.TryGetValue(toLabel, out var toSet); @@ -1208,6 +1214,8 @@ private static void EmitNormalizePairedStateOnEdge( ? $"{pad}{regExpr} = PPC_PsFromScalarInline({regExpr});" : $"{pad}{regExpr} = PPC_PsToScalarInline({regExpr});"); } + + _activeResidency?.EndEdge(); } private static int ParseFloatRegisterIndex(string name) diff --git a/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs new file mode 100644 index 000000000..b490c5945 --- /dev/null +++ b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs @@ -0,0 +1,621 @@ +using Translator.Core.Analysis; +using Translator.Core.Analysis.Representation; +using Translator.Core.Analysis.Ssa; +using Translator.Core.CodeGen; +using Translator.Core.Ir; +using Translator.Core.Representation; + +namespace Translator.Tests; + +/// +/// Register residency writes a resident register back to CpuContext only while its local may differ from the +/// context (written since the entry load, the last reload or the last write-back). +/// +public sealed class DirtyRegisterWritebackCodeGenTests +{ + private const string Lf = "\n"; + + private static string Emit(IrFunction function, RepresentationEnvironment? types = null) + { + return new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + types ?? new RepresentationEnvironment(new Dictionary())) + .ReplaceLineEndings(Lf); + } + + private static int Count(string text, string needle) => + text.Split(needle, StringSplitOptions.None).Length - 1; + + private static string BlockText(string code, string label) + { + var start = code.IndexOf($"\n{label}:\n", StringComparison.Ordinal); + Assert.True(start >= 0, $"missing block {label}"); + var end = code.IndexOf("\n}\n", start, StringComparison.Ordinal); + Assert.True(end > start, $"unterminated block {label}"); + return code[start..end]; + } + + private static IrInstruction Inc(string reg) => + new IrBinary(reg, IrValue.Register(reg), IrValue.Imm(1), "add"); + + [Fact] + public void StraightLineWritesBackOnlyRegistersChangedSinceTheLastSync() + { + // r3 and r4 are modified before the call, so the call flushes both. Afterwards only r3 changes, so the + // return must publish r3 and nothing else: r4 was reloaded by the call and is equal to CpuContext. + var function = new IrFunction("dirty_straight", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + Inc("r4"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r3"), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(2, Count(code, "ctx->gpr[3] = r3;")); + Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.DoesNotContain("ctx->gpr[4] = r4;", code[call..], StringComparison.Ordinal); + } + + [Fact] + public void CleanRegisterIsNotPublishedAgainAfterABoundaryReloadedIt() + { + var function = new IrFunction("dirty_clean_after_call", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.Contains("r3 = ctx->gpr[3];", code, StringComparison.Ordinal); + } + + [Fact] + public void RegisterWrittenOnlyAfterTheLastBoundaryIsNotFlushedAtThatBoundary() + { + var function = new IrFunction("dirty_late_write", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r5"), + new IrReturn(null) + }) + }); + + var code = Emit(function); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + + var beforeCall = code[..call]; + Assert.Contains("ctx->gpr[3] = r3;", beforeCall, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[5] = r5;", beforeCall, StringComparison.Ordinal); + var afterCall = code[call..]; + Assert.Contains("ctx->gpr[5] = r5;", afterCall, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[3] = r3;", afterCall, StringComparison.Ordinal); + } + + [Fact] + public void JoinOfDirtyAndCleanPathsKeepsTheRegisterDirty() + { + // entry: r3 modified. a: r5 modified, then a call (flushes + reloads everything). b: r6 modified. + // At merge: r3 and r6 are dirty on the path through b, r5 is clean on both paths. The return publishes r3 + // and r6 (not r5); the call in a publishes r3 and r5 (not r6, not yet modified there). + var function = new IrFunction("dirty_join", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrBranch("beq", "a", "b") + }), + new IrBasicBlock("a", new IrInstruction[] + { + Inc("r5"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("b", new IrInstruction[] + { + Inc("r6"), + new IrJump("merge") + }), + new IrBasicBlock("merge", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + var blockA = BlockText(code, "loc_a"); + Assert.Contains("ctx->gpr[3] = r3;", blockA, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[5] = r5;", blockA, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[6] = r6;", blockA, StringComparison.Ordinal); + + var merge = BlockText(code, "loc_merge"); + Assert.Contains("ctx->gpr[3] = r3;", merge, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[6] = r6;", merge, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[5] = r5;", merge, StringComparison.Ordinal); + } + + [Fact] + public void LoopBackEdgeCarriesDirtyStateIntoTheLoopHeader() + { + // r3 is modified inside the loop body, and the call in the header runs on every iteration: on the second + // iteration r3 is dirty at the header's call, so that call must publish it even though it is clean on + // the first entry. + var function = new IrFunction("dirty_loop", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrJump("head") + }), + new IrBasicBlock("head", new IrInstruction[] + { + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r3"), + new IrBranch("bne", "head", "exit") + }), + new IrBasicBlock("exit", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + var head = BlockText(code, "loc_head"); + var call = head.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.Contains("ctx->gpr[3] = r3;", head[..call], StringComparison.Ordinal); + } + + [Fact] + public void CleanFunctionEntryPublishesNothingAtAnExitThatFollowsNoWrite() + { + // r3 is written only on the "set" path; the "skip" path returns at once, publishing nothing. + var function = new IrFunction("dirty_skip", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrBranch("beq", "set", "skip") + }), + new IrBasicBlock("set", new IrInstruction[] + { + Inc("r3"), + new IrReturn(null) + }), + new IrBasicBlock("skip", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.DoesNotContain("ctx->gpr[3] = r3;", BlockText(code, "loc_skip"), StringComparison.Ordinal); + } + + [Fact] + public void ConditionRegisterAndCountRegisterAreTrackedLikeGprs() + { + var function = new IrFunction("dirty_cr_ctr", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrAssign("ctr", IrValue.Register("r3")), + new IrSetCrField(0, IrValue.Register("r3"), IrValue.Imm(0), false), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + // cr and ctr are published once (at the call) and reloaded; the return has nothing left to publish. + Assert.Equal(1, Count(code, "ctx->cr = cr;")); + Assert.Equal(1, Count(code, "ctx->ctr = ctr;")); + } + + [Fact] + public void IndirectJumpPublishesDirtyRegistersAndTrackingContinuesPastTheExit() + { + // The default arm of a jump table leaves the frame; its write-back must not make the register look clean + // on the case edges. + var function = new IrFunction("dirty_jump_table", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrJumpTable("r4", new[] + { + new IrJumpTableCase(0x80001010u, "t0"), + new IrJumpTableCase(0x80001020u, "t1") + }) + }), + new IrBasicBlock("t0", new IrInstruction[] + { + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }), + new IrBasicBlock("t1", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + // default arm, t0's call and t1's return each publish r3 (dirty from entry on every path). + Assert.Equal(3, Count(code, "ctx->gpr[3] = r3;")); + Assert.Contains("ctx->gpr[3] = r3;", BlockText(code, "loc_t1"), StringComparison.Ordinal); + } + + [Fact] + public void EdgeNormalizationWriteMakesTheRegisterDirtyOnThatEdgeOnly() + { + // f5 is produced as a paired value, flushed and reloaded by a call (clean), then normalized to scalar on the + // edge into "merge". The edge write makes f5 dirty at merge's return; the other predecessor leaves it clean. + var function = new IrFunction("dirty_edge", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrBranch("beq", "paired", "scalar") + }), + new IrBasicBlock("paired", new IrInstruction[] + { + new IrCall("f5", "PPC_PsMul", new[] { IrValue.Register("f1"), IrValue.Register("f2") }), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("scalar", new IrInstruction[] + { + new IrAssign("f5", IrValue.Register("f3")), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("merge", new IrInstruction[] + { + new IrCall("f6", "PPC_PsNeg", new[] { IrValue.Register("f5") }), + new IrReturn(null) + }) + }); + var types = new RepresentationEnvironment(new Dictionary + { + ["f1"] = ValueRepresentation.Float64, + ["f2"] = ValueRepresentation.Float64, + ["f3"] = ValueRepresentation.Float64, + ["f5"] = ValueRepresentation.Float64, + ["f6"] = ValueRepresentation.Float64, + ["cr0"] = ValueRepresentation.UInt32 + }); + + var code = Emit(function, types); + + Assert.Contains("f5.d = PPC_PsToScalarInline(f5.d);", code, StringComparison.Ordinal); + var merge = BlockText(code, "loc_merge"); + // f5 reached merge clean on both paths, and normalization wrote it on one edge: dirty, so published. + Assert.Contains("ctx->fpr[5] = f5;", merge, StringComparison.Ordinal); + // f6 is produced in merge itself. + Assert.Contains("ctx->fpr[6] = f6;", merge, StringComparison.Ordinal); + } + + [Fact] + public void IndirectCallIsAFullBoundaryAndLeavesEveryRegisterClean() + { + var function = new IrFunction("dirty_indirect_call", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + Inc("r4"), + new IrIndirectCall(string.Empty, IrValue.Register("r5"), Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + } + + [Fact] + public void FunctionWithLrContinuationLabelsKeepsPublishingEveryWrittenRegister() + { + // The callee may resume this body at the continuation label with state the dataflow does not model, so a + // function that has one stays on the old write-back-everything-written behaviour. + var function = new IrFunction("dirty_lr_continuation", "0x800E591C", new[] + { + new IrBasicBlock("0x800E591C", new IrInstruction[] + { + new IrAssign("r4", IrValue.Imm(5)), + new IrAssign("lr", IrValue.Imm(unchecked((int)0x800E5920u))), + new IrCall(string.Empty, "0x8179AC3C", Array.Empty()), + new IrAssign("r3", IrValue.Imm(8)), + new IrReturn(null) + }), + new IrBasicBlock("0x800E5934", new IrInstruction[] + { + new IrAssign("r3", IrValue.Imm(1)), + new IrReturn(null) + }) + }); + var types = new RepresentationEnvironment(new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r3"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32 + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x800E591C, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + types, + lrContinuationCallTargets: new HashSet { 0x8179AC3Cu }).ReplaceLineEndings(Lf); + + var continuation = BlockText(code, "loc_800E5934"); + Assert.Contains("ctx->gpr[3] = r3;", continuation, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[4] = r4;", continuation, StringComparison.Ordinal); + } + + private static GuestAbiContract Contract(uint gprRead = 0, uint gprWrite = 0) => + new( + GprReadBeforeWriteMask: gprRead, + GprPossibleWriteMask: gprWrite, + GprReturnMask: 0, + FprReadBeforeWriteMask: 0, + FprPossibleWriteMask: 0, + FprReturnMask: 0, + CrReadBeforeWriteMask: 0, + CrPossibleWriteMask: 0, + ReadsXerBeforeWrite: false, + MayWriteXer: false, + ReadsCtrBeforeWrite: false, + MayWriteCtr: false, + ReadsLrBeforeWrite: false, + MayWriteLr: false, + BoundaryFlags: GuestCallBoundaryFlags.None, + DirectCallTargets: Array.Empty()); + + [Fact] + public void NarrowContractFlushLeavesTheFlushedRegisterCleanWithoutAReload() + { + // The callee only reads r31: the call publishes r31 and does not reload it, so the return must not + // publish it a second time. r30 (written, outside the contract) stays dirty across the call. + var function = new IrFunction("dirty_narrow", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r31"), + Inc("r30"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 31) + }).ReplaceLineEndings(Lf); + + Assert.Equal(1, Count(code, "ctx->gpr[31] = r31;")); + Assert.Equal(1, Count(code, "ctx->gpr[30] = r30;")); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.DoesNotContain("ctx->gpr[31] = r31;", code[call..], StringComparison.Ordinal); + Assert.Contains("ctx->gpr[30] = r30;", code[call..], StringComparison.Ordinal); + } + + [Fact] + public void VolatileRegisterIsPublishedAgainAfterABoundaryThatDidNotReloadIt() + { + // The first callee only reads r3, so its flush leaves r3 equal to CpuContext. The second declares + // nothing, but a native there can run a guest handler on this context that changes r3; the return + // has to republish the resident r3, as it would without dirty tracking. + var function = new IrFunction("dirty_volatile_callback", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 3), + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var second = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(second > 0); + Assert.Contains("ctx->gpr[3] = r3;", code[second..], StringComparison.Ordinal); + } + + [Fact] + public void R13ReloadedByAFullBoundaryIsPublishedAgainAfterANarrowOne() + { + // The alarm pump behind a narrow native resets CpuContext's r13 to the SDA base. + var function = new IrFunction("dirty_r13_reload", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r13"), + new IrCall(string.Empty, "0x80004000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var narrow = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(narrow > 0); + Assert.Contains("r13 = ctx->gpr[13];", code[..narrow], StringComparison.Ordinal); + Assert.Contains("ctx->gpr[13] = r13;", code[narrow..], StringComparison.Ordinal); + } + + [Fact] + public void XerReloadedByAFullBoundaryIsPublishedAgainAfterANarrowOne() + { + var function = new IrFunction("dirty_xer_reload", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrCall("xer", "PPC_UpdateCarrySub", new[] { IrValue.Register("r3"), IrValue.Register("r4") }), + new IrCall(string.Empty, "0x80004000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + // 0x80004000 has no contract: a full boundary that reloads XER. + var narrow = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(narrow > 0); + Assert.Contains("xer = ctx->xer;", code[..narrow], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[narrow..], StringComparison.Ordinal); + } + + [Fact] + public void WrittenXerIsPublishedAgainAtEveryBoundaryThatDoesNotReloadIt() + { + // Neither callee writes XER, so neither call reloads it. A handler run behind such a contract can still + // change CpuContext's XER, which is why every boundary republishes the resident one. + var function = new IrFunction("dirty_xer", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrCall("xer", "PPC_UpdateCarrySub", new[] { IrValue.Register("r3"), IrValue.Register("r4") }), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(), + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var first = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + var second = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(first > 0 && second > first); + Assert.Contains("ctx->xer = xer;", code[..first], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[first..second], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[second..], StringComparison.Ordinal); + } + + [Fact] + public void StateFreeFastArmResultStaysDirtyAfterTheSlowArmCleanedIt() + { + // Fast arm: the result lands in the r3 local only (dirty). Slow arm: flush + call + reload (clean). After + // the join r3 is dirty, so the return publishes it; r4 was dirty before the call and is not touched by + // the fast arm, so it is dirty too. + var function = new IrFunction("dirty_state_free", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r4"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + stateFreeAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 3, gprWrite: 1u << 3) + }, + stateFreeCallSymbols: new Dictionary { [0x80002000u] = "callee_native" }) + .ReplaceLineEndings(Lf); + + var elseArm = code.IndexOf("} else {", StringComparison.Ordinal); + var slowEnd = code.IndexOf("\n }\n", elseArm, StringComparison.Ordinal); + Assert.True(elseArm > 0 && slowEnd > elseArm, code); + var afterCall = code[slowEnd..]; + Assert.Contains("ctx->gpr[3] = r3;", afterCall, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[4] = r4;", afterCall, StringComparison.Ordinal); + // The slow arm itself publishes what is dirty at that point (r4 only: r3 is not yet written there). + var slowArm = code[elseArm..slowEnd]; + Assert.Contains("ctx->gpr[4] = r4;", slowArm, StringComparison.Ordinal); + } + + [Fact] + public void BlockNoRecordedEdgeReachesStartsWithEverythingDirty() + { + var function = new IrFunction("dirty_unreached", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrReturn(null) + }), + new IrBasicBlock("orphan", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Contains("ctx->gpr[3] = r3;", BlockText(code, "loc_orphan"), StringComparison.Ordinal); + } +}