port perf fixes from forks

runtime/include/guest_abi/dispatch.h

@@ -130,7 +130,6 @@ inline const RawDispatchRecord* FindStaticIndirectDispatchEntry(
 }
 
 void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* table);
-inline std::atomic<const StaticIndirectDispatchTable*> g_publishedStaticIndirectDispatchTable{nullptr};
 
 class StaticIndirectDispatchTableRegistrar {
 public:
@@ -298,8 +297,8 @@ struct IsPureTypedNativeCpuCall<T, std::void_t<decltype(T::kPure)>> : std::bool_
 
 #include "guest_abi/native_cpu_calls.inc"
 
-// Direct-mapped thread-local memo in front of the sorted registry lookup every bctrl performs;
-// a miss just re-runs the sorted lookup. 512 entries (12 KiB) covers the per-frame indirect
+// Direct-mapped thread-local memo in front of the registry lookup for targets without a raw
+// dispatch record; a miss just re-runs the lookup. 512 entries (12 KiB) covers the per-frame indirect
 // working set while staying L1/L2 resident, unlike 4096 which would thrash L2. Must stay a
 // power of two, the index masks with (size - 1).
 inline constexpr size_t kIndirectDispatchCacheEntries = 512;
@@ -314,13 +313,23 @@ struct IndirectResolvedDispatchMemoEntry {
 inline thread_local IndirectResolvedDispatchMemoEntry
     g_indirectResolvedDispatchMemo[kIndirectDispatchCacheEntries]{};
 
-struct IndirectRawDispatchMemoEntry {
-    bool valid;
+// Every raw dispatch record in one open-addressed table, built by Finalize and shared by all
+// threads. Kept at most half full, so a lookup almost always reads a single slot, and unlike a
+// per-thread memo it never misses into the generated table's page search.
+struct RawDispatchHashSlot {
     uint32_t address;
-    const RawDispatchRecord* record;
+    const RawDispatchRecord* record; // nullptr = empty
+};
+
+struct RawDispatchHashTable {
+    const RawDispatchHashSlot* slots;
+    uint32_t mask;  // slot count - 1
+    uint32_t shift; // 32 - log2(slot count); Fibonacci hashing keeps the product's top bits
 };
-inline thread_local IndirectRawDispatchMemoEntry
-    g_indirectRawDispatchMemo[kIndirectDispatchCacheEntries]{};
+
+MKW_PPC_FORCE_INLINE uint32_t RawDispatchHashIndex(const RawDispatchHashTable& table, uint32_t address) {
+    return ((address >> 2) * 2654435761u) >> table.shift;
+}
 
 class TranslatedFunctionRegistry {
 public:
@@ -345,48 +354,29 @@ class TranslatedFunctionRegistry {
         return info;
     }
 
-    // Hot wrapper: only the direct-mapped memo probe stays inline. Inlining the
-    // generated-table binary search alongside it pushed the whole function past
-    // the inliner's budget and out-of-lined the probe itself, which is the part
-    // every bctrl executes.
+    // nullptr before Finalize, like the publication flag this replaced.
     static MKW_PPC_FORCE_INLINE const RawDispatchRecord* FindRawByAddressPtr(uint32_t address) {
-        if (!lookupPublished_.load(std::memory_order_acquire)) {
+        const RawDispatchHashTable* table = rawDispatchTable_.load(std::memory_order_acquire);
+        if (table == nullptr) {
             return nullptr;
         }
-
-        auto& cached = g_indirectRawDispatchMemo[
-            ((address >> 2) * 2654435761u) & (kIndirectDispatchCacheEntries - 1)];
-        if (cached.valid && cached.address == address) {
-            return cached.record;
+        for (uint32_t index = RawDispatchHashIndex(*table, address);; index = (index + 1) & table->mask) {
+            const RawDispatchHashSlot& slot = table->slots[index];
+            if (slot.record == nullptr || slot.address == address) {
+                return slot.record;
+            }
         }
-
-        return FindRawByAddressPtrMiss(address, cached);
     }
 
 private:
-    // Memo miss: resolve against the header-inline generated table first; only dynamically
-    // registered records or a not-yet-published table fall through to the slow path.
-    static MKW_PPC_NO_INLINE MKW_PPC_COLD const RawDispatchRecord* FindRawByAddressPtrMiss(
-        uint32_t address, IndirectRawDispatchMemoEntry& cached) {
-        const RawDispatchRecord* record = nullptr;
-        if (const auto* table =
-                g_publishedStaticIndirectDispatchTable.load(std::memory_order_acquire)) {
-            record = FindStaticIndirectDispatchEntry(table, address);
-        }
-        if (record == nullptr) {
-            record = FindRawByAddressPtrSlow(address);
-        }
-        cached = {true, address, record};
-        return record;
-    }
-
     static const TranslatedFunctionInfo* FindByAddressPtrSlow(uint32_t address);
-    static const RawDispatchRecord* FindRawByAddressPtrSlow(uint32_t address);
+    static void BuildRawDispatchTableLocked();
 
     // Finalize publishes immutable registry, generated-table, and dynamic raw
     // dispatch storage. Header-inlined cache hits are enabled only after that
     // release publication, so a pre-finalization miss can never poison them.
     inline static std::atomic_bool lookupPublished_{false};
+    inline static std::atomic<const RawDispatchHashTable*> rawDispatchTable_{nullptr};
 };
 
 // Cold, profile-owned registration data is emitted in compact table shards.

runtime/include/isa/ppc_isa_float.h

@@ -191,7 +191,7 @@ inline bool MkwHostNiActiveInline() noexcept
     return g_mkwHostNiActive;
 }
 
-inline float PpcForceSingleValueInline(double value)
+inline float PpcForceSingleValueSlowInline(double value)
 {
     // FPSCR[NI] flushes an exact pre-round single-subnormal even when rounding would promote it
     // to the smallest normal. g_mkwNiFlushThreshold (2^-126 active, 0.0 inactive) turns the
@@ -216,6 +216,18 @@ inline float PpcForceSingleValueInline(double value)
 #endif
 }
 
+inline float PpcForceSingleValueInline(double value)
+{
+    // Convert first and take the exact pre-round check above only for a +-FLT_MIN or subnormal
+    // result. A larger magnitude means |value| was at least 2^-126 (no rounding mode carries a
+    // smaller one past FLT_MIN), +-0 is what the flush produces anyway, and NaN/infinity never
+    // flush, so every other result matches. This keeps the mask off each single op's dependency chain.
+    const float converted = static_cast<float>(value);
+    if ((PpcBitCastToU32Inline(converted) & 0x7FFFFFFFu) - 1u < 0x00800000u) [[unlikely]]
+        return PpcForceSingleValueSlowInline(value);
+    return converted;
+}
+
 inline float PpcFlushSingleForNiInline(float value)
 {
     if (!MkwHostNiActiveInline())

runtime/src/abi_bridge.cpp

@@ -307,9 +307,6 @@ void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* tabl
         std::abort();
     }
     registered = table;
-    // Mirror into the header-visible atomic under the same lock, so the
-    // inlined miss path and this owning static can never disagree.
-    g_publishedStaticIndirectDispatchTable.store(table, std::memory_order_release);
 }
 
 void RegisterBulkTranslatedFunctions(const BulkTranslatedFunctionRecord* records, size_t count) {
@@ -398,6 +395,7 @@ void TranslatedFunctionRegistry::Finalize() {
             auto& addressEntries = AddressEntries();
             addressEntries.clear();
             addressEntries.rehash(0);
+            BuildRawDispatchTableLocked();
             RegistryFrozen().store(true, std::memory_order_release);
             lookupPublished_.store(true, std::memory_order_release);
         }
@@ -478,19 +476,39 @@ GuestFunctions::Entry GuestFunctions::NextTranslation(uint32_t address) {
     ReportMissingGuestFunction(address);
 }
 
-const RawDispatchRecord* TranslatedFunctionRegistry::FindRawByAddressPtrSlow(uint32_t address) {
-    if (!RegistryFrozen().load(std::memory_order_acquire)) {
-        return nullptr;
+void TranslatedFunctionRegistry::BuildRawDispatchTableLocked() {
+    const auto* generated = GeneratedIndirectDispatchTable();
+    const auto& dynamic = DynamicRawDispatchEntries();
+    const size_t generatedCount = generated != nullptr ? generated->entryCount : 0;
+    uint32_t bits = 10;
+    while ((size_t{1} << bits) < (generatedCount + dynamic.size()) * 2) {
+        ++bits;
+    }
+    static std::vector<RawDispatchHashSlot> slots;
+    static RawDispatchHashTable table;
+    slots.assign(size_t{1} << bits, RawDispatchHashSlot{0, nullptr});
+    table = {slots.data(), (1u << bits) - 1u, 32u - bits};
+    // The first record put at an address wins: the generated table, then the dynamic records in
+    // address order, the precedence the sorted lookups gave.
+    const auto put = [](const RawDispatchRecord& record) {
+        for (uint32_t index = RawDispatchHashIndex(table, record.address);; index = (index + 1) & table.mask) {
+            RawDispatchHashSlot& slot = slots[index];
+            if (slot.record == nullptr) {
+                slot = {record.address, &record};
+                return;
+            }
+            if (slot.address == record.address) {
+                return;
+            }
+        }
+    };
+    for (size_t i = 0; i < generatedCount; ++i) {
+        put(generated->entries[i]);
     }
-    if (const auto* generated = FindStaticIndirectDispatchEntry(GeneratedIndirectDispatchTable(), address)) {
-        return generated;
+    for (const auto& record : dynamic) {
+        put(record);
     }
-    const auto& entries = DynamicRawDispatchEntries();
-    const auto it = std::lower_bound(entries.begin(), entries.end(), address,
-        [](const RawDispatchRecord& entry, uint32_t target) {
-            return entry.address < target;
-        });
-    return it != entries.end() && it->address == address ? &*it : nullptr;
+    rawDispatchTable_.store(&table, std::memory_order_release);
 }
 
 std::optional<TranslatedFunctionInfo> FindTranslatedFunctionByHostAddress(uintptr_t hostAddr) {

runtime/src/hle/gx/gx_copy.cpp

@@ -8,6 +8,7 @@
 #include <algorithm>
 #include <map>
 #include <mutex>
+#include <unordered_map>
 #include <vector>
 
 namespace {
@@ -20,12 +21,47 @@ std::mutex g_efbCopyDestinationsMutex;
 std::map<uint32_t, uint32_t> g_efbCopyDestinations;
 uint32_t g_largestEfbCopyDestination = 0;
 
+// How many destinations touch each 64 KiB bucket of guest RAM. Nearly every DCStoreRange misses all
+// destinations, and an empty bucket answers that without walking the map.
+constexpr uint32_t kEfbCopyBucketShift = 16;
+std::unordered_map<uint32_t, uint32_t> g_efbCopyBuckets;
+
+void CountEfbCopyBucketsLocked(uint64_t start, uint32_t size, int32_t delta) {
+    const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift;
+    for (uint64_t bucket = start >> kEfbCopyBucketShift; bucket <= last; ++bucket) {
+        g_efbCopyBuckets[static_cast<uint32_t>(bucket)] += static_cast<uint32_t>(delta);
+    }
+}
+
+bool AnyEfbCopyBucketLocked(uint64_t start, uint32_t size) {
+    const uint64_t first = start >> kEfbCopyBucketShift;
+    const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift;
+    if (last - first + 1u > g_efbCopyBuckets.size()) {
+        return true; // wider than the index; the map walk answers
+    }
+    for (uint64_t bucket = first; bucket <= last; ++bucket) {
+        const auto it = g_efbCopyBuckets.find(static_cast<uint32_t>(bucket));
+        if (it != g_efbCopyBuckets.end() && it->second != 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void RememberEfbCopyDestination(uint32_t addr, uint32_t size) {
     if (size == 0) {
         return;
     }
     std::lock_guard<std::mutex> guard(g_efbCopyDestinationsMutex);
-    g_efbCopyDestinations[CanonicalizeGxMainRamAddress(addr)] = size;
+    const uint32_t canonical = CanonicalizeGxMainRamAddress(addr);
+    const auto [it, inserted] = g_efbCopyDestinations.try_emplace(canonical, size);
+    if (inserted) {
+        CountEfbCopyBucketsLocked(canonical, size, +1);
+    } else if (it->second != size) {
+        CountEfbCopyBucketsLocked(canonical, it->second, -1);
+        CountEfbCopyBucketsLocked(canonical, size, +1);
+        it->second = size;
+    }
     // A conservative monotonic maximum lets invalidation jump directly to
     // the only map interval that could overlap instead of walking every copy
     // destination on every DCStoreRange.
@@ -44,6 +80,9 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) {
     std::vector<uint32_t> retired;
     {
         std::lock_guard<std::mutex> guard(g_efbCopyDestinationsMutex);
+        if (!AnyEfbCopyBucketLocked(dirtyStart, size)) {
+            return;
+        }
         const uint64_t earliestCandidate =
             dirtyStart > g_largestEfbCopyDestination ? dirtyStart - g_largestEfbCopyDestination : 0;
         for (auto it = g_efbCopyDestinations.lower_bound(static_cast<uint32_t>(earliestCandidate));
@@ -55,6 +94,7 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) {
                 continue;
             }
             retired.push_back(it->first);
+            CountEfbCopyBucketsLocked(it->first, it->second, -1);
             it = g_efbCopyDestinations.erase(it);
         }
     }

runtime/src/hle/gx/gx_dl.cpp

@@ -1207,6 +1207,27 @@ static void SubmitLytDrawPacket(const uint8_t* packet, uint32_t packetBytes) {
     SyncAppliedVtxStateFromHleReal();
 }
 
+// A span inside one fully mapped 1 MiB page resolves straight from the page bias, the pointer
+// Contains + GuestToHostPtr would give after two table lookups.
+static const uint8_t* ResolveDisplayListArray(uint32_t base, uint32_t span) {
+#if !defined(__APPLE__)
+    // macOS has to keep GuestToHostPtr's canonical alias, which is the GX resource identity there.
+    if (base != 0 && span != 0) {
+        const uint32_t page = base >> MemoryInline::kPageShift;
+        const uint32_t last = base + span - 1u;
+        if (last >= base && (last >> MemoryInline::kPageShift) == page) {
+            if (const uintptr_t bias = MemoryInline::g_fullPageBias[page]; bias != 0) {
+                return reinterpret_cast<const uint8_t*>((bias - 1u) + base);
+            }
+        }
+    }
+#endif
+    if (!Memory::Contains(base, span)) {
+        return nullptr;
+    }
+    return static_cast<const uint8_t*>(GuestToHostPtr(base, span));
+}
+
 static bool ApplyAuroraArraysForDisplayList(const std::array<uint32_t, GX_VA_MAX_ATTR>& maxIdx,
                                             const std::array<bool, GX_VA_MAX_ATTR>& sawIdx,
                                             GXVtxFmt vtxfmt,
@@ -1232,11 +1253,7 @@ static bool ApplyAuroraArraysForDisplayList(const std::array<uint32_t, GX_VA_MAX
         }
         const uint32_t span = (count > 0) ? (static_cast<uint32_t>((count - 1u) * arr.stride) + elemSize) : 0u;
         const bool needsPack = arr.stride > 0xffu;
-        if (!Memory::Contains(arr.base, span)) {
-            ok = false;
-            continue;
-        }
-        const uint8_t* hostPtr = static_cast<const uint8_t*>(GuestToHostPtr(arr.base, span));
+        const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span);
         if (!hostPtr) {
             ok = false;
             continue;
@@ -1273,11 +1290,7 @@ static bool ApplyAuroraIndexedXFArraysForDisplayList(const std::array<uint32_t,
             continue;
         }
         const uint32_t span = maxXfIdx[attr] * arr.stride + bytes;
-        if (!Memory::Contains(arr.base, span)) {
-            ok = false;
-            continue;
-        }
-        const uint8_t* hostPtr = static_cast<const uint8_t*>(GuestToHostPtr(arr.base, span));
+        const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span);
         if (!hostPtr) {
             ok = false;
             continue;

runtime/src/hle/gx/gx_fifo.cpp

@@ -343,9 +343,39 @@ void SubmitAttribute(GXAttr attr, float* comps, const VtxAttrFmt& fmt, const u32
     }
 }
 
+// A word that lands inside an XF load still short of its last word only appends: the general
+// path below would push it and then stop parsing at the same incomplete packet.
+static bool TryAppendIncompleteXfWord(u32 value) {
+    auto& g = g_hleGxState;
+    if (g.inBegin || g.fifoByteCount < 5u ||
+        g.fifoReadOffset + g.fifoByteCount + 4u > g.fifoBytes.size()) {
+        return false;
+    }
+    uint8_t* data = g.fifoBytes.data() + g.fifoReadOffset;
+    if ((data[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) {
+        return false;
+    }
+    const uint32_t packetBytes = 5u + (static_cast<uint32_t>(ReadBE16(data + 1)) + 1u) * 4u;
+    if (g.fifoByteCount + 4u >= packetBytes) {
+        return false;
+    }
+    BigEndian::Write32(data + g.fifoByteCount, value);
+    g.fifoByteCount += 4u;
+    return true;
+}
+
+static MKW_PPC_NO_INLINE void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes);
+
 // `val` is a raw big-endian bit pattern: the FIFO stream is type-agnostic, and
 // the float entry point converts before it gets here.
 void HleFifoWrite(u32 val, uint32_t sizeBytes) {
+    if ((sizeBytes == 0 || sizeBytes == 4) && !IsDisplayListActive() && TryAppendIncompleteXfWord(val)) {
+        return;
+    }
+    HleFifoWriteGeneral(val, sizeBytes);
+}
+
+static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) {
     const bool recordOnly = IsDisplayListActive();
     if (recordOnly) {
         WriteDisplayListData(val, sizeBytes);
@@ -727,9 +757,19 @@ static uint32_t ApplyFifoPacketsDirect(const uint8_t* data, uint32_t sizeBytes)
             const uint16_t countWords = ReadBE16(packet + 1);
             const uint32_t packetBytes = 1u + 4u + (static_cast<uint32_t>(countWords) + 1u) * 4u;
             if (avail < packetBytes) break;
-            GXCallDisplayList(packet, packetBytes);
+            // Aurora parses a run of adjacent XF loads in one call. The run stays inside this
+            // burst, so it keeps its order against the CP/BP writes around it.
+            uint32_t runBytes = packetBytes;
+            while (avail - runBytes >= 5u) {
+                const uint8_t* next = packet + runBytes;
+                if ((next[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) break;
+                const uint32_t nextBytes = 5u + (static_cast<uint32_t>(ReadBE16(next + 1)) + 1u) * 4u;
+                if (avail - runBytes < nextBytes) break;
+                runBytes += nextBytes;
+            }
+            GXCallDisplayList(packet, runBytes);
             GXMarkFrameWork();
-            offset += packetBytes;
+            offset += runBytes;
             continue;
         }