From f2f5390abf6ab2feb38b2fe1a812d7d4f6d8cfd8 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:33:03 +0100 Subject: [PATCH 01/18] convert singles before the ni check --- runtime/include/isa/ppc_isa_float.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/runtime/include/isa/ppc_isa_float.h b/runtime/include/isa/ppc_isa_float.h index 4aa6f4eb1..c01377145 100644 --- a/runtime/include/isa/ppc_isa_float.h +++ b/runtime/include/isa/ppc_isa_float.h @@ -191,7 +191,7 @@ inline bool MkwHostNiActiveInline() noexcept return g_mkwHostNiActive; } -inline float PpcForceSingleValueInline(double value) +inline float PpcForceSingleValueSlowInline(double value) { // FPSCR[NI] flushes an exact pre-round single-subnormal even when rounding would promote it // to the smallest normal. g_mkwNiFlushThreshold (2^-126 active, 0.0 inactive) turns the @@ -216,6 +216,18 @@ inline float PpcForceSingleValueInline(double value) #endif } +inline float PpcForceSingleValueInline(double value) +{ + // Convert first and take the exact pre-round check above only for a +-FLT_MIN or subnormal + // result. A larger magnitude means |value| was at least 2^-126 (no rounding mode carries a + // smaller one past FLT_MIN), +-0 is what the flush produces anyway, and NaN/infinity never + // flush, so every other result matches. This keeps the mask off each single op's dependency chain. + const float converted = static_cast(value); + if ((PpcBitCastToU32Inline(converted) & 0x7FFFFFFFu) - 1u < 0x00800000u) [[unlikely]] + return PpcForceSingleValueSlowInline(value); + return converted; +} + inline float PpcFlushSingleForNiInline(float value) { if (!MkwHostNiActiveInline()) From 09d6c8efcbb940313cc3b2cf2cb1254e184f73ce Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:33:42 +0100 Subject: [PATCH 02/18] bulk copy dvd reads into writable pages --- runtime/src/hle/storage/dvd.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/runtime/src/hle/storage/dvd.cpp b/runtime/src/hle/storage/dvd.cpp index b76e5e19c..2c55c46eb 100644 --- a/runtime/src/hle/storage/dvd.cpp +++ b/runtime/src/hle/storage/dvd.cpp @@ -103,10 +103,23 @@ extern "C" void DVDInit_8015EA1C(); extern "C" uint32_t g_dvdFstReservedBase; extern "C" uint32_t g_dvdFstReservedSize; -// Byte-wise copy into guest RAM plus the DMA notification the GX caches need. +// Copy into guest RAM plus the DMA notification the GX caches need. A fully writable 1 MiB page +// takes its bytes in one memcpy, since Write8 is a plain store there; any other page keeps Write8. static void CopyToGuestAsDma(uint32_t dest, const uint8_t* data, size_t size) { - for (size_t i = 0; i < size; ++i) { - Memory::Write8(dest + static_cast(i), data[i]); + size_t done = 0; + while (done < size) { + const uint32_t address = dest + static_cast(done); + const size_t pageLeft = MemoryInline::kPageSize - (address & MemoryInline::kPageMask); + const size_t chunk = std::min(size - done, pageLeft); + if (const uintptr_t bias = MemoryInline::g_fullWritablePageBias[address >> MemoryInline::kPageShift]; + bias != 0) { + std::memcpy(reinterpret_cast((bias - 1u) + address), data + done, chunk); + } else { + for (size_t i = 0; i < chunk; ++i) { + Memory::Write8(address + static_cast(i), data[done + i]); + } + } + done += chunk; } GxNotifyGuestRamDmaWrite(dest, static_cast(size)); } From 0991e06eb8010832e5aefe2a0b5d1ac7489f5354 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:36:18 +0100 Subject: [PATCH 03/18] write back only dirty registers --- .../CxxLinearCodeGenerator.EmitInstruction.cs | 2 +- ...xxLinearCodeGenerator.RegisterResidency.cs | 280 +++++++++- .../CxxLinearCodeGenerator.StateFreeAbi.cs | 5 + .../CodeGen/CxxLinearCodeGenerator.cs | 18 +- .../DirtyRegisterWritebackCodeGenTests.cs | 491 ++++++++++++++++++ 5 files changed, 776 insertions(+), 20 deletions(-) create mode 100644 translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs index c5d3cd95f..a561e0a89 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs @@ -560,7 +560,7 @@ _activeResidency is not null && // Tail dispatch: the callee inherits authoritative context // and this frame never observes registers again, so the // locals must not be written back afterwards. - AppendFlush(sb, pad); + AppendExitFlush(sb, pad); sb.AppendLine($"{pad}InvokeIndirectJump({jumpTarget}, ctx);"); sb.AppendLine($"{pad}return;"); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs index 9c03dbacc..4ecec87cc 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs @@ -26,13 +26,93 @@ private sealed class RegisterResidency private bool _writesCtr; private bool _writesXer; - private RegisterResidency(bool recording) => _recording = recording; + // Dirty-register tracking (CpuContext write-back only for registers whose local changed since the last sync). + // A register is dirty while its local may differ from CpuContext: set by a write, cleared by an entry load, + // a reload after a boundary, or a write-back. The recording pass summarizes every block as a gen/kill + // transfer plus its outgoing edge writes; Snapshot() solves the forward union dataflow over them; the + // real pass replays the same transfer from the solved block-entry state at every flush point. + // A function that can resume through a local LR-continuation label has edges this analysis does not + // see, so it keeps the old write-back-everything-written behaviour. + private readonly string? _entryLabel; + private readonly bool _conservative; + private Dictionary? _blocks; + private Dictionary? _blockIn; + private BlockSummary? _activeBlock; + private DirtySet _activeIn; + private string? _edgeTarget; + private DirtySet _edgeGen; + private readonly Stack<(Transfer Base, List Done)> _alternatives = new(); + + private const byte CrBit = 1; + private const byte CtrBit = 2; + private const byte XerBit = 4; + + internal struct DirtySet : IEquatable + { + public uint Gpr; + public uint Fpr; + public byte Flags; + + public static DirtySet Or(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr | b.Gpr, Fpr = a.Fpr | b.Fpr, Flags = (byte)(a.Flags | b.Flags) }; + + public static DirtySet And(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr & b.Gpr, Fpr = a.Fpr & b.Fpr, Flags = (byte)(a.Flags & b.Flags) }; + + public static DirtySet AndNot(DirtySet a, DirtySet b) => + new() { Gpr = a.Gpr & ~b.Gpr, Fpr = a.Fpr & ~b.Fpr, Flags = (byte)(a.Flags & ~b.Flags) }; + + public bool Equals(DirtySet other) => Gpr == other.Gpr && Fpr == other.Fpr && Flags == other.Flags; + public override bool Equals(object? obj) => obj is DirtySet other && Equals(other); + public override int GetHashCode() => HashCode.Combine(Gpr, Fpr, Flags); + } - public static RegisterResidency CreateRecorder() => new(true); + /// out = (in minus Kill) plus Gen. Closed under sequencing and under join of alternatives. + internal struct Transfer + { + public DirtySet Gen; + public DirtySet Kill; + + public readonly DirtySet Apply(DirtySet input) => DirtySet.Or(DirtySet.AndNot(input, Kill), Gen); + + public void Write(DirtySet written) + { + Gen = DirtySet.Or(Gen, written); + Kill = DirtySet.AndNot(Kill, written); + } + + public void Clean(DirtySet cleaned) + { + Gen = DirtySet.AndNot(Gen, cleaned); + Kill = DirtySet.Or(Kill, cleaned); + } + + public static Transfer Join(Transfer a, Transfer b) => new() + { + Gen = DirtySet.Or(a.Gen, b.Gen), + Kill = DirtySet.And(a.Kill, b.Kill) + }; + } + + private sealed class BlockSummary + { + public Transfer Transfer; + public readonly List<(string Target, DirtySet Gen)> Edges = new(); + } + + private RegisterResidency(bool recording, string? entryLabel = null, bool conservative = false) + { + _recording = recording; + _entryLabel = entryLabel; + _conservative = conservative; + } + + public static RegisterResidency CreateRecorder(string? entryLabel = null, bool conservative = false) => + new(true, entryLabel, conservative) { _blocks = new Dictionary(StringComparer.OrdinalIgnoreCase) }; public RegisterResidency Snapshot() { - var snapshot = new RegisterResidency(false); + var snapshot = new RegisterResidency(false, _entryLabel, _conservative); snapshot._gprs.UnionWith(_gprs); snapshot._fprs.UnionWith(_fprs); snapshot._writtenGprs.UnionWith(_writtenGprs); @@ -43,9 +123,143 @@ public RegisterResidency Snapshot() snapshot._writesCr = _writesCr; snapshot._writesCtr = _writesCtr; snapshot._writesXer = _writesXer; + snapshot._blocks = _blocks; + snapshot._blockIn = SolveDirtyEntryStates(snapshot.WrittenSet()); return snapshot; } + private DirtySet WrittenSet() + { + var set = new DirtySet(); + foreach (var gpr in _writtenGprs) set.Gpr |= 1u << gpr; + foreach (var fpr in _writtenFprs) set.Fpr |= 1u << fpr; + if (_writesCr) set.Flags |= CrBit; + if (_writesCtr) set.Flags |= CtrBit; + if (_writesXer) set.Flags |= XerBit; + return set; + } + + /// + /// Forward union dataflow over the recorded blocks. The entry block starts clean (entry loads); a block no + /// recorded edge reaches (and that is not the entry) starts with everything dirty, the old behaviour. + /// + private Dictionary? SolveDirtyEntryStates(DirtySet all) + { + if (_blocks is null || _conservative || _entryLabel is null || !_blocks.ContainsKey(_entryLabel)) + return null; + + var ins = new Dictionary(StringComparer.OrdinalIgnoreCase); + var reached = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var block in _blocks.Values) + foreach (var edge in block.Edges) + reached.Add(edge.Target); + foreach (var label in _blocks.Keys) + ins[label] = !string.Equals(label, _entryLabel, StringComparison.OrdinalIgnoreCase) && !reached.Contains(label) + ? all + : default; + + bool changed; + do + { + changed = false; + foreach (var (label, block) in _blocks) + { + var output = block.Transfer.Apply(ins[label]); + foreach (var (target, edgeGen) in block.Edges) + { + if (!ins.TryGetValue(target, out var current)) continue; + var merged = DirtySet.Or(current, DirtySet.Or(output, edgeGen)); + if (!merged.Equals(current)) + { + ins[target] = merged; + changed = true; + } + } + } + } + while (changed); + + return ins; + } + + /// Starts a block: the running transfer is empty and, in the real pass, its entry state is the solved one. + public void BeginBlock(string label) + { + _activeBlock = new BlockSummary(); + _edgeTarget = null; + _alternatives.Clear(); + if (_recording) + { + _blocks![label] = _activeBlock; + _activeIn = default; + } + else + { + _activeIn = _blockIn is not null && _blockIn.TryGetValue(label, out var known) ? known : WrittenSet(); + } + } + + /// Writes made while a CFG edge's normalization is emitted belong to that edge only. + public void BeginEdge(string target) + { + if (_activeBlock is null) return; + _edgeTarget = target; + _edgeGen = default; + } + + public void EndEdge() + { + if (_activeBlock is null || _edgeTarget is null) return; + if (_recording) _activeBlock.Edges.Add((_edgeTarget, _edgeGen)); + _edgeTarget = null; + } + + /// + /// Brackets code whose arms are alternatives within one block (a state-free call's fast and slow arm): + /// each arm starts from the state at ; afterwards the state is their join. + /// + public void BeginAlternatives() + { + if (_activeBlock is null) return; + _alternatives.Push((_activeBlock.Transfer, new List())); + } + + public void NextAlternative() + { + if (_activeBlock is null) return; + var (baseTransfer, done) = _alternatives.Peek(); + done.Add(_activeBlock.Transfer); + _activeBlock.Transfer = baseTransfer; + } + + public void EndAlternatives() + { + if (_activeBlock is null) return; + var (_, done) = _alternatives.Pop(); + var joined = _activeBlock.Transfer; + foreach (var arm in done) joined = Transfer.Join(joined, arm); + _activeBlock.Transfer = joined; + } + + private DirtySet CurrentDirty() + { + if (_activeBlock is null || _conservative) return WrittenSet(); + return _activeBlock.Transfer.Apply(_activeIn); + } + + private void NoteWrite(DirtySet written) + { + if (_activeBlock is null) return; + if (_edgeTarget is not null) _edgeGen = DirtySet.Or(_edgeGen, written); + else _activeBlock.Transfer.Write(written); + } + + private void NoteClean(DirtySet cleaned) + { + if (_activeBlock is null || _edgeTarget is not null) return; + _activeBlock.Transfer.Clean(cleaned); + } + public bool IsEmpty => _gprs.Count == 0 && _fprs.Count == 0 && !_cr && !_ctr && !_xer; public static string GprLocal(int index) => GprRegisterName(index); @@ -88,6 +302,7 @@ public string FprStorage(int index, bool written) public string Cr(bool written) { + if (written) NoteWrite(new DirtySet { Flags = CrBit }); if (_recording) { _cr = true; @@ -104,6 +319,7 @@ public string Cr(bool written) public string Ctr(bool written) { + if (written) NoteWrite(new DirtySet { Flags = CtrBit }); if (_recording) { _ctr = true; @@ -124,6 +340,7 @@ public string Ctr(bool written) /// public string Xer(bool written) { + if (written) NoteWrite(new DirtySet { Flags = XerBit }); if (_recording) { _xer = true; @@ -140,6 +357,12 @@ public string Xer(bool written) private void Track(SortedSet referenced, SortedSet written, int index, bool isWrite, string prefix) { + if (isWrite) + { + var bit = 1u << index; + NoteWrite(ReferenceEquals(referenced, _gprs) ? new DirtySet { Gpr = bit } : new DirtySet { Fpr = bit }); + } + if (_recording) { referenced.Add(index); @@ -183,18 +406,36 @@ public void AppendEntryLoads(StringBuilder sb) /// or overwrite are published; see /// for why the write set has to participate in the flush mask. /// - public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync) + public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync) => + AppendFlush(sb, pad, sync, leavesFrame: false); + + /// + /// Writes back the registers that are both dirty and in the boundary's flush set. A flush that stays in + /// this frame ( false) leaves them clean afterwards; one that returns or + /// tail-dispatches must not change the state the sibling paths of the block see. + /// + public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync, bool leavesFrame) { - if (_recording) return; - foreach (var gpr in _writtenGprs) - if (sync.FlushesGpr(gpr)) - sb.Append(pad).Append("ctx->gpr[").Append(gpr).Append("] = ").Append(GprLocal(gpr)).AppendLine(";"); - foreach (var fpr in _writtenFprs) - if (sync.FlushesFpr(fpr)) - sb.Append(pad).Append("ctx->fpr[").Append(fpr).Append("] = ").Append(FprLocal(fpr)).AppendLine(";"); - if (_writesCr && sync.FlushesCr) sb.Append(pad).AppendLine($"ctx->cr = {CrLocal};"); - if (_writesCtr && sync.FlushesCtr) sb.Append(pad).AppendLine($"ctx->ctr = {CtrLocal};"); - if (_writesXer && sync.FlushesXer) sb.Append(pad).AppendLine($"ctx->xer = {XerLocal};"); + var dirty = CurrentDirty(); + if (!_recording) + { + foreach (var gpr in _writtenGprs) + if (sync.FlushesGpr(gpr) && (dirty.Gpr & (1u << gpr)) != 0) + sb.Append(pad).Append("ctx->gpr[").Append(gpr).Append("] = ").Append(GprLocal(gpr)).AppendLine(";"); + foreach (var fpr in _writtenFprs) + if (sync.FlushesFpr(fpr) && (dirty.Fpr & (1u << fpr)) != 0) + sb.Append(pad).Append("ctx->fpr[").Append(fpr).Append("] = ").Append(FprLocal(fpr)).AppendLine(";"); + if (_writesCr && sync.FlushesCr && (dirty.Flags & CrBit) != 0) sb.Append(pad).AppendLine($"ctx->cr = {CrLocal};"); + if (_writesCtr && sync.FlushesCtr && (dirty.Flags & CtrBit) != 0) sb.Append(pad).AppendLine($"ctx->ctr = {CtrLocal};"); + if (_writesXer && sync.FlushesXer && (dirty.Flags & XerBit) != 0) sb.Append(pad).AppendLine($"ctx->xer = {XerLocal};"); + } + + if (leavesFrame) return; + var flushed = new DirtySet { Gpr = sync.FlushGprMask, Fpr = sync.FlushFprMask }; + if (sync.FlushesCr) flushed.Flags |= CrBit; + if (sync.FlushesCtr) flushed.Flags |= CtrBit; + if (sync.FlushesXer) flushed.Flags |= XerBit; + NoteClean(flushed); } /// @@ -206,6 +447,11 @@ public void AppendFlush(StringBuilder sb, string pad, ResidencyBoundarySync sync /// Contract-narrowed reload. public void AppendReload(StringBuilder sb, string pad, ResidencyBoundarySync sync) { + var reloaded = new DirtySet { Gpr = sync.ReloadGprMask, Fpr = sync.ReloadFprMask }; + if (sync.ReloadsCr) reloaded.Flags |= CrBit; + if (sync.ReloadsCtr) reloaded.Flags |= CtrBit; + if (sync.ReloadsXer) reloaded.Flags |= XerBit; + NoteClean(reloaded); if (_recording) return; foreach (var gpr in _gprs) if (sync.ReloadsGpr(gpr)) @@ -348,6 +594,12 @@ private static void AppendFlush(StringBuilder sb, string pad, ResidencyBoundaryS if (_activeResidency is { } residency) residency.AppendFlush(sb, pad, sync); } + /// Write-back on a path that leaves the frame (return, tail dispatch): publishes the dirty registers, tracks nothing. + private static void AppendExitFlush(StringBuilder sb, string pad) + { + if (_activeResidency is { } residency) residency.AppendFlush(sb, pad, ResidencyBoundarySync.Full, leavesFrame: true); + } + private static void AppendReload(StringBuilder sb, string pad, ResidencyBoundarySync sync) { if (_activeResidency is { } residency) residency.AppendReload(sb, pad, sync); diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs index 224b30334..f8adb3050 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.StateFreeAbi.cs @@ -375,6 +375,9 @@ private static void EmitResidentStateFreeCall( $"Resident state-free marshalling requested for 0x{address:X8} without an active residency."); AppendFlush(sb, pad, ResidencyBoundarySync.StateFreeResidentOuter); + // The fast arm's result stores and the slow arm's flush/reload are alternatives: each starts from + // the state here and the dirty set afterwards is their join. + residency.BeginAlternatives(); var inner = pad + " "; sb.AppendLine($"{pad}if ({StateFreeAvailabilityCondition(address, stateFreeAbiContracts, stateFreeCallSymbols)}) {{"); var arguments = StateFreeResidentCallArguments(residency, contract, callSiteLr); @@ -391,10 +394,12 @@ private static void EmitResidentStateFreeCall( sb.AppendLine($"{inner}{symbol}({arguments});"); } sb.AppendLine($"{pad}}} else {{"); + residency.NextAlternative(); AppendFlush(sb, inner, ResidencyBoundarySync.Full); RecordEmittedDirectCallTarget(address); sb.AppendLine($"{inner}InvokeDirectCpu<0x{address:X8}u>(ctx);"); AppendReload(sb, inner, ResidencyBoundarySync.Full); + residency.EndAlternatives(); sb.AppendLine($"{pad}}}"); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs index 2e0cee994..019d8dca6 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs @@ -434,6 +434,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool var localConstants = new Dictionary(StringComparer.OrdinalIgnoreCase); var label = labelNames[block.Label]; RecordEmittedLocalLabel(label); + _activeResidency?.BeginBlock(block.Label); body.AppendLine($"{label}:"); body.AppendLine("{"); var localPaired = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -523,12 +524,12 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool { case IrUndefined undef: EmitInstruction(block.Label, -1, undef, body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase: false, localFallthroughLr: null, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch, resumeLabelPairedStates); - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine(" return;"); break; case IrReturn ret: - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine(" return;"); break; @@ -576,7 +577,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool // Unlisted selectors leave this frame through the // dispatcher: publish the locals and never write // them back afterwards. - AppendFlush(body, " "); + AppendExitFlush(body, " "); body.AppendLine($" InvokeIndirectJump({selectorExpr}, ctx);"); body.AppendLine(" return;"); body.AppendLine(" }"); @@ -597,7 +598,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool // unreachable and must not write them back. if (term is not IrIndirectJump) { - AppendFlush(body, " "); + AppendExitFlush(body, " "); } body.AppendLine(" return;"); } @@ -647,7 +648,9 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool Environment.NewLine.Length; RegisterResidency? residency = null; { - var recorder = RegisterResidency.CreateRecorder(); + // A body that can resume through a local LR-continuation label has CFG edges the dirty-register + // analysis does not model, so it keeps the write-back-everything-written behaviour. + var recorder = RegisterResidency.CreateRecorder(func.EntryLabel, conservative: continuationCallCount > 0); EmitFunctionBody(recorder, bodyBufferBaseLength, discardResult: true); var discovered = recorder.Snapshot(); if (!discovered.IsEmpty) @@ -1170,6 +1173,9 @@ private static void EmitNormalizePairedStateOnEdge( IReadOnlyDictionary> pairedIn, IReadOnlyDictionary>? liveIn) { + // The edge's own register writes (normalization) and its target are what the dirty-register + // dataflow needs; every emitted CFG edge goes through here. + _activeResidency?.BeginEdge(toLabel); pairedOut.TryGetValue(fromLabel, out var fromSet); pairedIn.TryGetValue(toLabel, out var toSet); @@ -1208,6 +1214,8 @@ private static void EmitNormalizePairedStateOnEdge( ? $"{pad}{regExpr} = PPC_PsFromScalarInline({regExpr});" : $"{pad}{regExpr} = PPC_PsToScalarInline({regExpr});"); } + + _activeResidency?.EndEdge(); } private static int ParseFloatRegisterIndex(string name) diff --git a/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs new file mode 100644 index 000000000..4e27a8092 --- /dev/null +++ b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs @@ -0,0 +1,491 @@ +using Translator.Core.Analysis; +using Translator.Core.Analysis.Representation; +using Translator.Core.Analysis.Ssa; +using Translator.Core.CodeGen; +using Translator.Core.Ir; +using Translator.Core.Representation; + +namespace Translator.Tests; + +/// +/// Register residency writes a resident register back to CpuContext only while its local may differ from the +/// context (written since the entry load, the last reload or the last write-back). +/// +public sealed class DirtyRegisterWritebackCodeGenTests +{ + private const string Lf = "\n"; + + private static string Emit(IrFunction function, RepresentationEnvironment? types = null) + { + return new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + types ?? new RepresentationEnvironment(new Dictionary())) + .ReplaceLineEndings(Lf); + } + + private static int Count(string text, string needle) => + text.Split(needle, StringSplitOptions.None).Length - 1; + + private static string BlockText(string code, string label) + { + var start = code.IndexOf($"\n{label}:\n", StringComparison.Ordinal); + Assert.True(start >= 0, $"missing block {label}"); + var end = code.IndexOf("\n}\n", start, StringComparison.Ordinal); + Assert.True(end > start, $"unterminated block {label}"); + return code[start..end]; + } + + private static IrInstruction Inc(string reg) => + new IrBinary(reg, IrValue.Register(reg), IrValue.Imm(1), "add"); + + [Fact] + public void StraightLineWritesBackOnlyRegistersChangedSinceTheLastSync() + { + // r3 and r4 are modified before the call, so the call flushes both. Afterwards only r3 changes, so the + // return must publish r3 and nothing else: r4 was reloaded by the call and is equal to CpuContext. + var function = new IrFunction("dirty_straight", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + Inc("r4"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r3"), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(2, Count(code, "ctx->gpr[3] = r3;")); + Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.DoesNotContain("ctx->gpr[4] = r4;", code[call..], StringComparison.Ordinal); + } + + [Fact] + public void CleanRegisterIsNotPublishedAgainAfterABoundaryReloadedIt() + { + var function = new IrFunction("dirty_clean_after_call", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.Contains("r3 = ctx->gpr[3];", code, StringComparison.Ordinal); + } + + [Fact] + public void RegisterWrittenOnlyAfterTheLastBoundaryIsNotFlushedAtThatBoundary() + { + var function = new IrFunction("dirty_late_write", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r5"), + new IrReturn(null) + }) + }); + + var code = Emit(function); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + + var beforeCall = code[..call]; + Assert.Contains("ctx->gpr[3] = r3;", beforeCall, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[5] = r5;", beforeCall, StringComparison.Ordinal); + var afterCall = code[call..]; + Assert.Contains("ctx->gpr[5] = r5;", afterCall, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[3] = r3;", afterCall, StringComparison.Ordinal); + } + + [Fact] + public void JoinOfDirtyAndCleanPathsKeepsTheRegisterDirty() + { + // entry: r3 modified. a: r5 modified, then a call (flushes + reloads everything). b: r6 modified. + // At merge: r3 and r6 are dirty on the path through b, r5 is clean on both paths. The return publishes r3 + // and r6 (not r5); the call in a publishes r3 and r5 (not r6, not yet modified there). + var function = new IrFunction("dirty_join", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrBranch("beq", "a", "b") + }), + new IrBasicBlock("a", new IrInstruction[] + { + Inc("r5"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("b", new IrInstruction[] + { + Inc("r6"), + new IrJump("merge") + }), + new IrBasicBlock("merge", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + var blockA = BlockText(code, "loc_a"); + Assert.Contains("ctx->gpr[3] = r3;", blockA, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[5] = r5;", blockA, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[6] = r6;", blockA, StringComparison.Ordinal); + + var merge = BlockText(code, "loc_merge"); + Assert.Contains("ctx->gpr[3] = r3;", merge, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[6] = r6;", merge, StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[5] = r5;", merge, StringComparison.Ordinal); + } + + [Fact] + public void LoopBackEdgeCarriesDirtyStateIntoTheLoopHeader() + { + // r3 is modified inside the loop body, and the call in the header runs on every iteration: on the second + // iteration r3 is dirty at the header's call, so that call must publish it even though it is clean on + // the first entry. + var function = new IrFunction("dirty_loop", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrJump("head") + }), + new IrBasicBlock("head", new IrInstruction[] + { + new IrCall(string.Empty, "0x80002000", Array.Empty()), + Inc("r3"), + new IrBranch("bne", "head", "exit") + }), + new IrBasicBlock("exit", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + var head = BlockText(code, "loc_head"); + var call = head.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.Contains("ctx->gpr[3] = r3;", head[..call], StringComparison.Ordinal); + } + + [Fact] + public void CleanFunctionEntryPublishesNothingAtAnExitThatFollowsNoWrite() + { + // r3 is written only on the "set" path; the "skip" path returns at once, publishing nothing. + var function = new IrFunction("dirty_skip", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrBranch("beq", "set", "skip") + }), + new IrBasicBlock("set", new IrInstruction[] + { + Inc("r3"), + new IrReturn(null) + }), + new IrBasicBlock("skip", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.DoesNotContain("ctx->gpr[3] = r3;", BlockText(code, "loc_skip"), StringComparison.Ordinal); + } + + [Fact] + public void ConditionRegisterAndCountRegisterAreTrackedLikeGprs() + { + var function = new IrFunction("dirty_cr_ctr", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrAssign("ctr", IrValue.Register("r3")), + new IrSetCrField(0, IrValue.Register("r3"), IrValue.Imm(0), false), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + // cr and ctr are published once (at the call) and reloaded; the return has nothing left to publish. + Assert.Equal(1, Count(code, "ctx->cr = cr;")); + Assert.Equal(1, Count(code, "ctx->ctr = ctr;")); + } + + [Fact] + public void IndirectJumpPublishesDirtyRegistersAndTrackingContinuesPastTheExit() + { + // The default arm of a jump table leaves the frame; its write-back must not make the register look clean + // on the case edges. + var function = new IrFunction("dirty_jump_table", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrJumpTable("r4", new[] + { + new IrJumpTableCase(0x80001010u, "t0"), + new IrJumpTableCase(0x80001020u, "t1") + }) + }), + new IrBasicBlock("t0", new IrInstruction[] + { + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }), + new IrBasicBlock("t1", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + // default arm, t0's call and t1's return each publish r3 (dirty from entry on every path). + Assert.Equal(3, Count(code, "ctx->gpr[3] = r3;")); + Assert.Contains("ctx->gpr[3] = r3;", BlockText(code, "loc_t1"), StringComparison.Ordinal); + } + + [Fact] + public void EdgeNormalizationWriteMakesTheRegisterDirtyOnThatEdgeOnly() + { + // f5 is produced as a paired value, flushed and reloaded by a call (clean), then normalized to scalar on the + // edge into "merge". The edge write makes f5 dirty at merge's return; the other predecessor leaves it clean. + var function = new IrFunction("dirty_edge", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrBranch("beq", "paired", "scalar") + }), + new IrBasicBlock("paired", new IrInstruction[] + { + new IrCall("f5", "PPC_PsMul", new[] { IrValue.Register("f1"), IrValue.Register("f2") }), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("scalar", new IrInstruction[] + { + new IrAssign("f5", IrValue.Register("f3")), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrJump("merge") + }), + new IrBasicBlock("merge", new IrInstruction[] + { + new IrCall("f6", "PPC_PsNeg", new[] { IrValue.Register("f5") }), + new IrReturn(null) + }) + }); + var types = new RepresentationEnvironment(new Dictionary + { + ["f1"] = ValueRepresentation.Float64, + ["f2"] = ValueRepresentation.Float64, + ["f3"] = ValueRepresentation.Float64, + ["f5"] = ValueRepresentation.Float64, + ["f6"] = ValueRepresentation.Float64, + ["cr0"] = ValueRepresentation.UInt32 + }); + + var code = Emit(function, types); + + Assert.Contains("f5.d = PPC_PsToScalarInline(f5.d);", code, StringComparison.Ordinal); + var merge = BlockText(code, "loc_merge"); + // f5 reached merge clean on both paths, and normalization wrote it on one edge: dirty, so published. + Assert.Contains("ctx->fpr[5] = f5;", merge, StringComparison.Ordinal); + // f6 is produced in merge itself. + Assert.Contains("ctx->fpr[6] = f6;", merge, StringComparison.Ordinal); + } + + [Fact] + public void IndirectCallIsAFullBoundaryAndLeavesEveryRegisterClean() + { + var function = new IrFunction("dirty_indirect_call", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + Inc("r4"), + new IrIndirectCall(string.Empty, IrValue.Register("r5"), Array.Empty()), + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + } + + [Fact] + public void FunctionWithLrContinuationLabelsKeepsPublishingEveryWrittenRegister() + { + // The callee may resume this body at the continuation label with state the dataflow does not model, so a + // function that has one stays on the old write-back-everything-written behaviour. + var function = new IrFunction("dirty_lr_continuation", "0x800E591C", new[] + { + new IrBasicBlock("0x800E591C", new IrInstruction[] + { + new IrAssign("r4", IrValue.Imm(5)), + new IrAssign("lr", IrValue.Imm(unchecked((int)0x800E5920u))), + new IrCall(string.Empty, "0x8179AC3C", Array.Empty()), + new IrAssign("r3", IrValue.Imm(8)), + new IrReturn(null) + }), + new IrBasicBlock("0x800E5934", new IrInstruction[] + { + new IrAssign("r3", IrValue.Imm(1)), + new IrReturn(null) + }) + }); + var types = new RepresentationEnvironment(new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r3"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32 + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x800E591C, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + types, + lrContinuationCallTargets: new HashSet { 0x8179AC3Cu }).ReplaceLineEndings(Lf); + + var continuation = BlockText(code, "loc_800E5934"); + Assert.Contains("ctx->gpr[3] = r3;", continuation, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[4] = r4;", continuation, StringComparison.Ordinal); + } + + private static GuestAbiContract Contract(uint gprRead = 0, uint gprWrite = 0) => + new( + GprReadBeforeWriteMask: gprRead, + GprPossibleWriteMask: gprWrite, + GprReturnMask: 0, + FprReadBeforeWriteMask: 0, + FprPossibleWriteMask: 0, + FprReturnMask: 0, + CrReadBeforeWriteMask: 0, + CrPossibleWriteMask: 0, + ReadsXerBeforeWrite: false, + MayWriteXer: false, + ReadsCtrBeforeWrite: false, + MayWriteCtr: false, + ReadsLrBeforeWrite: false, + MayWriteLr: false, + BoundaryFlags: GuestCallBoundaryFlags.None, + DirectCallTargets: Array.Empty()); + + [Fact] + public void NarrowContractFlushLeavesTheFlushedRegisterCleanWithoutAReload() + { + // The callee only reads r3: the call publishes r3 and does not reload it, so the return must not publish + // it a second time. r4 (written, outside the contract) stays dirty across the call. + var function = new IrFunction("dirty_narrow", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + Inc("r4"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 3) + }).ReplaceLineEndings(Lf); + + Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); + Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + Assert.DoesNotContain("ctx->gpr[3] = r3;", code[call..], StringComparison.Ordinal); + Assert.Contains("ctx->gpr[4] = r4;", code[call..], StringComparison.Ordinal); + } + + [Fact] + public void StateFreeFastArmResultStaysDirtyAfterTheSlowArmCleanedIt() + { + // Fast arm: the result lands in the r3 local only (dirty). Slow arm: flush + call + reload (clean). After + // the join r3 is dirty, so the return publishes it; r4 was dirty before the call and is not touched by + // the fast arm, so it is dirty too. + var function = new IrFunction("dirty_state_free", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r4"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + stateFreeAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 3, gprWrite: 1u << 3) + }, + stateFreeCallSymbols: new Dictionary { [0x80002000u] = "callee_native" }) + .ReplaceLineEndings(Lf); + + var elseArm = code.IndexOf("} else {", StringComparison.Ordinal); + var slowEnd = code.IndexOf("\n }\n", elseArm, StringComparison.Ordinal); + Assert.True(elseArm > 0 && slowEnd > elseArm, code); + var afterCall = code[slowEnd..]; + Assert.Contains("ctx->gpr[3] = r3;", afterCall, StringComparison.Ordinal); + Assert.Contains("ctx->gpr[4] = r4;", afterCall, StringComparison.Ordinal); + // The slow arm itself publishes what is dirty at that point (r4 only: r3 is not yet written there). + var slowArm = code[elseArm..slowEnd]; + Assert.Contains("ctx->gpr[4] = r4;", slowArm, StringComparison.Ordinal); + } + + [Fact] + public void BlockNoRecordedEdgeReachesStartsWithEverythingDirty() + { + var function = new IrFunction("dirty_unreached", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrReturn(null) + }), + new IrBasicBlock("orphan", new IrInstruction[] + { + new IrReturn(null) + }) + }); + + var code = Emit(function); + + Assert.Contains("ctx->gpr[3] = r3;", BlockText(code, "loc_orphan"), StringComparison.Ordinal); + } +} From 187a1b4781193db02cf4555ce5ef518dd2dbf4db Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:40:11 +0100 Subject: [PATCH 04/18] shared hash table for indirect calls --- runtime/include/guest_abi/dispatch.h | 64 ++++++++++++---------------- runtime/src/abi_bridge.cpp | 46 ++++++++++++++------ 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/runtime/include/guest_abi/dispatch.h b/runtime/include/guest_abi/dispatch.h index dba9ef15e..d038033e6 100644 --- a/runtime/include/guest_abi/dispatch.h +++ b/runtime/include/guest_abi/dispatch.h @@ -130,7 +130,6 @@ inline const RawDispatchRecord* FindStaticIndirectDispatchEntry( } void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* table); -inline std::atomic g_publishedStaticIndirectDispatchTable{nullptr}; class StaticIndirectDispatchTableRegistrar { public: @@ -298,8 +297,8 @@ struct IsPureTypedNativeCpuCall> : std::bool_ #include "guest_abi/native_cpu_calls.inc" -// Direct-mapped thread-local memo in front of the sorted registry lookup every bctrl performs; -// a miss just re-runs the sorted lookup. 512 entries (12 KiB) covers the per-frame indirect +// Direct-mapped thread-local memo in front of the registry lookup for targets without a raw +// dispatch record; a miss just re-runs the lookup. 512 entries (12 KiB) covers the per-frame indirect // working set while staying L1/L2 resident, unlike 4096 which would thrash L2. Must stay a // power of two, the index masks with (size - 1). inline constexpr size_t kIndirectDispatchCacheEntries = 512; @@ -314,13 +313,23 @@ struct IndirectResolvedDispatchMemoEntry { inline thread_local IndirectResolvedDispatchMemoEntry g_indirectResolvedDispatchMemo[kIndirectDispatchCacheEntries]{}; -struct IndirectRawDispatchMemoEntry { - bool valid; +// Every raw dispatch record in one open-addressed table, built by Finalize and shared by all +// threads. Kept at most half full, so a lookup almost always reads a single slot, and unlike a +// per-thread memo it never misses into the generated table's page search. +struct RawDispatchHashSlot { uint32_t address; - const RawDispatchRecord* record; + const RawDispatchRecord* record; // nullptr = empty +}; + +struct RawDispatchHashTable { + const RawDispatchHashSlot* slots; + uint32_t mask; // slot count - 1 + uint32_t shift; // 32 - log2(slot count); Fibonacci hashing keeps the product's top bits }; -inline thread_local IndirectRawDispatchMemoEntry - g_indirectRawDispatchMemo[kIndirectDispatchCacheEntries]{}; + +MKW_PPC_FORCE_INLINE uint32_t RawDispatchHashIndex(const RawDispatchHashTable& table, uint32_t address) { + return ((address >> 2) * 2654435761u) >> table.shift; +} class TranslatedFunctionRegistry { public: @@ -345,48 +354,29 @@ class TranslatedFunctionRegistry { return info; } - // Hot wrapper: only the direct-mapped memo probe stays inline. Inlining the - // generated-table binary search alongside it pushed the whole function past - // the inliner's budget and out-of-lined the probe itself, which is the part - // every bctrl executes. + // nullptr before Finalize, like the publication flag this replaced. static MKW_PPC_FORCE_INLINE const RawDispatchRecord* FindRawByAddressPtr(uint32_t address) { - if (!lookupPublished_.load(std::memory_order_acquire)) { + const RawDispatchHashTable* table = rawDispatchTable_.load(std::memory_order_acquire); + if (table == nullptr) { return nullptr; } - - auto& cached = g_indirectRawDispatchMemo[ - ((address >> 2) * 2654435761u) & (kIndirectDispatchCacheEntries - 1)]; - if (cached.valid && cached.address == address) { - return cached.record; + for (uint32_t index = RawDispatchHashIndex(*table, address);; index = (index + 1) & table->mask) { + const RawDispatchHashSlot& slot = table->slots[index]; + if (slot.record == nullptr || slot.address == address) { + return slot.record; + } } - - return FindRawByAddressPtrMiss(address, cached); } private: - // Memo miss: resolve against the header-inline generated table first; only dynamically - // registered records or a not-yet-published table fall through to the slow path. - static MKW_PPC_NO_INLINE MKW_PPC_COLD const RawDispatchRecord* FindRawByAddressPtrMiss( - uint32_t address, IndirectRawDispatchMemoEntry& cached) { - const RawDispatchRecord* record = nullptr; - if (const auto* table = - g_publishedStaticIndirectDispatchTable.load(std::memory_order_acquire)) { - record = FindStaticIndirectDispatchEntry(table, address); - } - if (record == nullptr) { - record = FindRawByAddressPtrSlow(address); - } - cached = {true, address, record}; - return record; - } - static const TranslatedFunctionInfo* FindByAddressPtrSlow(uint32_t address); - static const RawDispatchRecord* FindRawByAddressPtrSlow(uint32_t address); + static void BuildRawDispatchTableLocked(); // Finalize publishes immutable registry, generated-table, and dynamic raw // dispatch storage. Header-inlined cache hits are enabled only after that // release publication, so a pre-finalization miss can never poison them. inline static std::atomic_bool lookupPublished_{false}; + inline static std::atomic rawDispatchTable_{nullptr}; }; // Cold, profile-owned registration data is emitted in compact table shards. diff --git a/runtime/src/abi_bridge.cpp b/runtime/src/abi_bridge.cpp index 55fe41651..6c7c67cd6 100644 --- a/runtime/src/abi_bridge.cpp +++ b/runtime/src/abi_bridge.cpp @@ -307,9 +307,6 @@ void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* tabl std::abort(); } registered = table; - // Mirror into the header-visible atomic under the same lock, so the - // inlined miss path and this owning static can never disagree. - g_publishedStaticIndirectDispatchTable.store(table, std::memory_order_release); } void RegisterBulkTranslatedFunctions(const BulkTranslatedFunctionRecord* records, size_t count) { @@ -398,6 +395,7 @@ void TranslatedFunctionRegistry::Finalize() { auto& addressEntries = AddressEntries(); addressEntries.clear(); addressEntries.rehash(0); + BuildRawDispatchTableLocked(); RegistryFrozen().store(true, std::memory_order_release); lookupPublished_.store(true, std::memory_order_release); } @@ -478,19 +476,39 @@ GuestFunctions::Entry GuestFunctions::NextTranslation(uint32_t address) { ReportMissingGuestFunction(address); } -const RawDispatchRecord* TranslatedFunctionRegistry::FindRawByAddressPtrSlow(uint32_t address) { - if (!RegistryFrozen().load(std::memory_order_acquire)) { - return nullptr; +void TranslatedFunctionRegistry::BuildRawDispatchTableLocked() { + const auto* generated = GeneratedIndirectDispatchTable(); + const auto& dynamic = DynamicRawDispatchEntries(); + const size_t generatedCount = generated != nullptr ? generated->entryCount : 0; + uint32_t bits = 10; + while ((size_t{1} << bits) < (generatedCount + dynamic.size()) * 2) { + ++bits; + } + static std::vector slots; + static RawDispatchHashTable table; + slots.assign(size_t{1} << bits, RawDispatchHashSlot{0, nullptr}); + table = {slots.data(), (1u << bits) - 1u, 32u - bits}; + // The first record put at an address wins: the generated table, then the dynamic records in + // address order, the precedence the sorted lookups gave. + const auto put = [](const RawDispatchRecord& record) { + for (uint32_t index = RawDispatchHashIndex(table, record.address);; index = (index + 1) & table.mask) { + RawDispatchHashSlot& slot = slots[index]; + if (slot.record == nullptr) { + slot = {record.address, &record}; + return; + } + if (slot.address == record.address) { + return; + } + } + }; + for (size_t i = 0; i < generatedCount; ++i) { + put(generated->entries[i]); } - if (const auto* generated = FindStaticIndirectDispatchEntry(GeneratedIndirectDispatchTable(), address)) { - return generated; + for (const auto& record : dynamic) { + put(record); } - const auto& entries = DynamicRawDispatchEntries(); - const auto it = std::lower_bound(entries.begin(), entries.end(), address, - [](const RawDispatchRecord& entry, uint32_t target) { - return entry.address < target; - }); - return it != entries.end() && it->address == address ? &*it : nullptr; + rawDispatchTable_.store(&table, std::memory_order_release); } std::optional FindTranslatedFunctionByHostAddress(uintptr_t hostAddr) { From 245e74fdb3cab7c4e927c152a32a915b2bf71f5a Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:41:14 +0100 Subject: [PATCH 05/18] cache register-only display lists --- runtime/src/hle/gx/gx_dl.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index daceeb785..c9254a174 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1320,6 +1320,13 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) const bool mayContainDraw = (cached != nullptr) ? cached->mayContainDraw : DisplayListMayContainDraw(list, nbytes); if (!mayContainDraw) { + // Register-only lists are remembered too, so the next call skips the walk. + if (cached == nullptr && allowScanCache) { + const uint64_t contentDigest = + probe.digestValid ? probe.contentDigest : DlContentDigest(list, nbytes); + StoreDlScanCache(listAddr, nbytes, scanLayoutHash, contentDigest, probe.writeGeneration, + false, DlScanCacheEntry{}, {}, nullptr, 0); + } EnsureAuroraFrameActive(); GXMarkFrameWork(); GXCallDisplayList(list, nbytes); From a9ad0763cfef2076c6619f5bc755b165c5435d17 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:41:44 +0100 Subject: [PATCH 06/18] batch adjacent xf loads --- runtime/src/hle/gx/gx_fifo.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/runtime/src/hle/gx/gx_fifo.cpp b/runtime/src/hle/gx/gx_fifo.cpp index 108e4fd55..adcb2754b 100644 --- a/runtime/src/hle/gx/gx_fifo.cpp +++ b/runtime/src/hle/gx/gx_fifo.cpp @@ -727,9 +727,19 @@ static uint32_t ApplyFifoPacketsDirect(const uint8_t* data, uint32_t sizeBytes) const uint16_t countWords = ReadBE16(packet + 1); const uint32_t packetBytes = 1u + 4u + (static_cast(countWords) + 1u) * 4u; if (avail < packetBytes) break; - GXCallDisplayList(packet, packetBytes); + // Aurora parses a run of adjacent XF loads in one call. The run stays inside this + // burst, so it keeps its order against the CP/BP writes around it. + uint32_t runBytes = packetBytes; + while (avail - runBytes >= 5u) { + const uint8_t* next = packet + runBytes; + if ((next[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) break; + const uint32_t nextBytes = 5u + (static_cast(ReadBE16(next + 1)) + 1u) * 4u; + if (avail - runBytes < nextBytes) break; + runBytes += nextBytes; + } + GXCallDisplayList(packet, runBytes); GXMarkFrameWork(); - offset += packetBytes; + offset += runBytes; continue; } From a1a93acebe364622f9f978209dc4840cb6964489 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:45:28 +0100 Subject: [PATCH 07/18] skip parsing for partial xf words and resolve dl arrays from the page bias --- runtime/src/hle/gx/gx_dl.cpp | 33 +++++++++++++++++++++++---------- runtime/src/hle/gx/gx_fifo.cpp | 30 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index c9254a174..a53584e65 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1207,6 +1207,27 @@ static void SubmitLytDrawPacket(const uint8_t* packet, uint32_t packetBytes) { SyncAppliedVtxStateFromHleReal(); } +// A span inside one fully mapped 1 MiB page resolves straight from the page bias, the pointer +// Contains + GuestToHostPtr would give after two table lookups. +static const uint8_t* ResolveDisplayListArray(uint32_t base, uint32_t span) { +#if !defined(__APPLE__) + // macOS has to keep GuestToHostPtr's canonical alias, which is the GX resource identity there. + if (base != 0 && span != 0) { + const uint32_t page = base >> MemoryInline::kPageShift; + const uint32_t last = base + span - 1u; + if (last >= base && (last >> MemoryInline::kPageShift) == page) { + if (const uintptr_t bias = MemoryInline::g_fullPageBias[page]; bias != 0) { + return reinterpret_cast((bias - 1u) + base); + } + } + } +#endif + if (!Memory::Contains(base, span)) { + return nullptr; + } + return static_cast(GuestToHostPtr(base, span)); +} + static bool ApplyAuroraArraysForDisplayList(const std::array& maxIdx, const std::array& sawIdx, GXVtxFmt vtxfmt, @@ -1232,11 +1253,7 @@ static bool ApplyAuroraArraysForDisplayList(const std::array 0) ? (static_cast((count - 1u) * arr.stride) + elemSize) : 0u; const bool needsPack = arr.stride > 0xffu; - if (!Memory::Contains(arr.base, span)) { - ok = false; - continue; - } - const uint8_t* hostPtr = static_cast(GuestToHostPtr(arr.base, span)); + const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span); if (!hostPtr) { ok = false; continue; @@ -1273,11 +1290,7 @@ static bool ApplyAuroraIndexedXFArraysForDisplayList(const std::array(GuestToHostPtr(arr.base, span)); + const uint8_t* hostPtr = ResolveDisplayListArray(arr.base, span); if (!hostPtr) { ok = false; continue; diff --git a/runtime/src/hle/gx/gx_fifo.cpp b/runtime/src/hle/gx/gx_fifo.cpp index adcb2754b..f122bd292 100644 --- a/runtime/src/hle/gx/gx_fifo.cpp +++ b/runtime/src/hle/gx/gx_fifo.cpp @@ -343,9 +343,39 @@ void SubmitAttribute(GXAttr attr, float* comps, const VtxAttrFmt& fmt, const u32 } } +// A word that lands inside an XF load still short of its last word only appends: the general +// path below would push it and then stop parsing at the same incomplete packet. +static bool TryAppendIncompleteXfWord(u32 value) { + auto& g = g_hleGxState; + if (g.inBegin || g.fifoByteCount < 5u || + g.fifoReadOffset + g.fifoByteCount + 4u > g.fifoBytes.size()) { + return false; + } + uint8_t* data = g.fifoBytes.data() + g.fifoReadOffset; + if ((data[0] & GX_OPCODE_MASK_CMD) != GX_LOAD_XF_REG_CMD) { + return false; + } + const uint32_t packetBytes = 5u + (static_cast(ReadBE16(data + 1)) + 1u) * 4u; + if (g.fifoByteCount + 4u >= packetBytes) { + return false; + } + BigEndian::Write32(data + g.fifoByteCount, value); + g.fifoByteCount += 4u; + return true; +} + +static MKW_PPC_NO_INLINE void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes); + // `val` is a raw big-endian bit pattern: the FIFO stream is type-agnostic, and // the float entry point converts before it gets here. void HleFifoWrite(u32 val, uint32_t sizeBytes) { + if ((sizeBytes == 0 || sizeBytes == 4) && !IsDisplayListActive() && TryAppendIncompleteXfWord(val)) { + return; + } + HleFifoWriteGeneral(val, sizeBytes); +} + +static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { const bool recordOnly = IsDisplayListActive(); if (recordOnly) { WriteDisplayListData(val, sizeBytes); From c49f14d8f3331981a3ca58c10ba740e99830528c Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:46:21 +0100 Subject: [PATCH 08/18] decode yaz0 through host pointers --- runtime/src/hle/egg_decomp.cpp | 81 ++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/runtime/src/hle/egg_decomp.cpp b/runtime/src/hle/egg_decomp.cpp index 94ab9ebec..7fcdc2a22 100644 --- a/runtime/src/hle/egg_decomp.cpp +++ b/runtime/src/hle/egg_decomp.cpp @@ -1,20 +1,19 @@ #include "hle_stubs.h" #include +#include #include "memory.h" +#include "recomp_mod_loader.h" #include "runtime_log.h" +extern "C" void GxNotifyGuestRamDmaWrite(uint32_t addr, uint32_t size); + // Native because a crafted Yaz0 run writes past the caller's buffer // (github.com/vabold/szsHaxx) // https://github.com/vabold/Kinoko/blob/main/source/egg/core/Decomp.cc -extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) -{ - const uint32_t expandSize = (static_cast(MemoryInline::FlatRead8(src + 4)) << 24) | - (static_cast(MemoryInline::FlatRead8(src + 5)) << 16) | - (static_cast(MemoryInline::FlatRead8(src + 6)) << 8) | - static_cast(MemoryInline::FlatRead8(src + 7)); - +template +static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { uint32_t srcIdx = 16; uint32_t dstIdx = 0; uint32_t mask = 0; @@ -22,15 +21,15 @@ extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) while (static_cast(dstIdx) < static_cast(expandSize)) { if (mask == 0) { - flags = MemoryInline::FlatRead8(src + srcIdx++); + flags = access.ReadSource(srcIdx++); mask = 0x80; } if ((flags & mask) != 0) { - MemoryInline::FlatWrite8(dst + dstIdx++, MemoryInline::FlatRead8(src + srcIdx++)); + access.WriteOutput(dstIdx++, access.ReadSource(srcIdx++)); } else { - const uint32_t high = MemoryInline::FlatRead8(src + srcIdx); - const uint32_t low = MemoryInline::FlatRead8(src + srcIdx + 1); + const uint32_t high = access.ReadSource(srcIdx); + const uint32_t low = access.ReadSource(srcIdx + 1); srcIdx += 2; const uint32_t rep = (high << 8) | low; @@ -48,7 +47,7 @@ extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) uint32_t count = rep >> 12; count = count != 0 ? count + 2 - : static_cast(MemoryInline::FlatRead8(src + srcIdx++)) + 18; + : static_cast(access.ReadSource(srcIdx++)) + 18; for (uint32_t i = 0; i < count; ++i) { if (dstIdx >= expandSize) { @@ -59,7 +58,7 @@ extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) "The game stopped decoding a malformed Yaz0 file."); std::abort(); } - MemoryInline::FlatWrite8(dst + dstIdx++, MemoryInline::FlatRead8(dst + copyIdx++)); + access.WriteOutput(dstIdx++, access.ReadOutput(copyIdx++)); } } @@ -69,5 +68,61 @@ extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) return expandSize; } +struct GuestSZSAccess { + uint32_t src; + uint32_t dst; + + uint8_t ReadSource(uint32_t offset) const { return MemoryInline::FlatRead8(src + offset); } + uint8_t ReadOutput(uint32_t offset) const { return MemoryInline::FlatRead8(dst + offset); } + void WriteOutput(uint32_t offset, uint8_t value) const { + MemoryInline::FlatWrite8(dst + offset, value); + } +}; + +struct HostSZSAccess { + const uint8_t* src; + uint8_t* dst; + + uint8_t ReadSource(uint32_t offset) const { return src[offset]; } + uint8_t ReadOutput(uint32_t offset) const { return dst[offset]; } + void WriteOutput(uint32_t offset, uint8_t value) const { dst[offset] = value; } +}; + +static bool HasDeferredReadPages(uint32_t address, size_t length) { + const uint32_t first = address >> MemoryInline::kPageShift; + const uint32_t last = static_cast( + (static_cast(address) + length - 1) >> MemoryInline::kPageShift); + for (uint32_t page = first; page <= last; ++page) { + if (MemoryInline::g_deferredReadCoveredPages[page] != 0) return true; + } + return false; +} + +extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) +{ + const uint32_t expandSize = (static_cast(MemoryInline::FlatRead8(src + 4)) << 24) | + (static_cast(MemoryInline::FlatRead8(src + 5)) << 16) | + (static_cast(MemoryInline::FlatRead8(src + 6)) << 8) | + static_cast(MemoryInline::FlatRead8(src + 7)); + + // A token produces at least one byte; all-literal data is the largest + // valid input (one flag per eight output bytes). Three extra bytes cover + // the final malformed run before its output-overrun check aborts. + const size_t maxSourceBytes = 16ull + expandSize + (static_cast(expandSize) + 7) / 8 + 3; + if (expandSize != 0 && expandSize <= static_cast(std::numeric_limits::max()) && + !GuestFlat::RequiresCheckedAccess() && + Memory::Contains(src, maxSourceBytes) && Memory::Contains(dst, expandSize) && + !HasDeferredReadPages(src, maxSourceBytes) && !HasDeferredReadPages(dst, expandSize) && + !RecompMod::ExecutableWriteGuardMayHit(dst, expandSize)) { + HostSZSAccess access{Memory::GetPointer(src, maxSourceBytes), Memory::GetPointer(dst, expandSize)}; + const uint32_t decoded = DecodeSZS(src, expandSize, access); + GxNotifyGuestRamDmaWrite(dst, decoded); + return decoded; + } + + GuestSZSAccess access{src, dst}; + return DecodeSZS(src, expandSize, access); +} + PPC_NATIVE_OVERRIDE(80218C2C, EGG_Decomp_decodeSZS_80218c2c, uint32_t, (uint32_t src, uint32_t dst), (src, dst)); From 99a5b03e3c645abb7030f71855652f58bda26bc6 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:46:21 +0100 Subject: [PATCH 09/18] keep dvd file handles open --- runtime/include/hle/dvd_contract.h | 69 +++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/runtime/include/hle/dvd_contract.h b/runtime/include/hle/dvd_contract.h index 9cb9576a6..8f4dcf456 100644 --- a/runtime/include/hle/dvd_contract.h +++ b/runtime/include/hle/dvd_contract.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -294,14 +295,57 @@ inline bool ReadExact(const std::filesystem::path& hostPath, HostReadFailure& failure) { failure = HostReadFailure::None; - std::ifstream file(hostPath, std::ios::binary); - if (!file.is_open()) { - failure = HostReadFailure::MissingFile; - return false; + // DVD callbacks run on the guest thread, often reading the same stream + // file many times per second. Reuse a bounded set of handles without + // sharing seek positions with another host thread. + struct OpenFile { + std::filesystem::path path; + std::ifstream stream; + std::streamoff size = 0; + std::filesystem::file_time_type modified{}; + uint64_t lastUse = 0; + }; + static thread_local std::vector openFiles; + static thread_local uint64_t useCount = 0; + constexpr size_t kMaxOpenFiles = 8; + + auto found = std::find_if(openFiles.begin(), openFiles.end(), + [&](const OpenFile& entry) { return entry.path == hostPath; }); + if (found != openFiles.end()) { + std::error_code ec; + const auto modified = std::filesystem::last_write_time(hostPath, ec); + if (ec || modified != found->modified) { + openFiles.erase(found); + found = openFiles.end(); + } + } + + if (found == openFiles.end()) { + std::ifstream file(hostPath, std::ios::binary); + if (!file.is_open()) { + failure = HostReadFailure::MissingFile; + return false; + } + file.seekg(0, std::ios::end); + const std::streamoff size = file.tellg(); + if (size < 0) { + failure = HostReadFailure::BadOffset; + return false; + } + std::error_code ec; + const auto modified = std::filesystem::last_write_time(hostPath, ec); + if (openFiles.size() == kMaxOpenFiles) { + const auto oldest = std::min_element(openFiles.begin(), openFiles.end(), + [](const OpenFile& a, const OpenFile& b) { return a.lastUse < b.lastUse; }); + openFiles.erase(oldest); + } + openFiles.push_back({hostPath, std::move(file), size, + ec ? std::filesystem::file_time_type{} : modified, 0}); + found = std::prev(openFiles.end()); } + found->lastUse = ++useCount; - file.seekg(0, std::ios::end); - const std::streamoff fileSize = file.tellg(); + const std::streamoff fileSize = found->size; if (fileSize < 0 || offset > static_cast(std::numeric_limits::max()) || offset >= static_cast(fileSize)) { @@ -316,17 +360,20 @@ inline bool ReadExact(const std::filesystem::path& hostPath, } std::vector staged(length); - file.seekg(static_cast(offset), std::ios::beg); - if (!file) { + found->stream.clear(); + found->stream.seekg(static_cast(offset), std::ios::beg); + if (!found->stream) { failure = HostReadFailure::BadOffset; + openFiles.erase(found); return false; } if (length != 0) { - file.read(reinterpret_cast(staged.data()), - static_cast(length)); - if (file.gcount() != static_cast(length)) { + found->stream.read(reinterpret_cast(staged.data()), + static_cast(length)); + if (found->stream.gcount() != static_cast(length)) { failure = HostReadFailure::ShortRead; + openFiles.erase(found); return false; } } From 8c7e048bca51efeffd7c1ae1c34fbc985b84406e Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:46:46 +0100 Subject: [PATCH 10/18] bucket the texture dirty index --- runtime/src/hle/gx/gx_internal.h | 6 +- runtime/src/hle/gx/gx_objects.cpp | 218 ++++++++++++------------------ 2 files changed, 91 insertions(+), 133 deletions(-) diff --git a/runtime/src/hle/gx/gx_internal.h b/runtime/src/hle/gx/gx_internal.h index b8270a9b9..98dcc3a38 100644 --- a/runtime/src/hle/gx/gx_internal.h +++ b/runtime/src/hle/gx/gx_internal.h @@ -154,9 +154,9 @@ struct TexObjSlot : TexObjMeta { uint64_t guestShadow[4] = {0, 0, 0, 0}; bool hasGuestShadow = false; - // DCStoreRange interval-index bookkeeping: what this slot currently - // contributes to g_texObjIntervals, so a pending update that turns out to - // be a no-op costs a compare instead of an erase + insert. + // DCStoreRange interval-index bookkeeping: the range this slot is listed + // under in g_texObjBuckets, so a pending update that turns out to be a + // no-op costs a compare instead of an unlist + relist. uint32_t indexedDataAddr = 0; uint32_t indexedSize = 0; bool inIntervalIndex = false; diff --git a/runtime/src/hle/gx/gx_objects.cpp b/runtime/src/hle/gx/gx_objects.cpp index 645658dfb..7e1f554ed 100644 --- a/runtime/src/hle/gx/gx_objects.cpp +++ b/runtime/src/hle/gx/gx_objects.cpp @@ -1,8 +1,11 @@ #include "gx_internal.h" #include "runtime_log.h" +#include #include #include +#include +#include // --- Texture and TLUT Objects --- @@ -84,21 +87,17 @@ struct TexObjIntervalEntry { TexObjSlot* slot = nullptr; }; -// DCStoreRange invalidation index, sorted by (dataAddr, objAddr). Rebuilding is O(N log N) plus a -// GXGetTexBufferSize per texobj, so it must not happen once per DCStoreRange: mutations instead -// queue their slot for a targeted patch. The queue used to be a capacity-limited vector that -// overflowed under real per-frame GXLoadTexObj volume and silently degraded into a full rebuild -// every call; the dedupe flag now lives on the slot so queuing is O(1) and uncapped. -std::vector g_texObjIntervals; +// DCStoreRange invalidation index: every indexed slot is listed in each 64 KiB bucket of guest RAM +// its texture data overlaps (slot.indexedDataAddr/indexedSize), so a flushed range only looks at +// the slots in its own buckets. It replaced a vector sorted by data address, which shifted the whole +// index whenever a texture object moved (memmove was 0.7% of the game thread on a slow Android device, in +// courses), and walked back over every entry within the largest texture's size of each flush. +// Mutations queue their slot (an O(1) dedupe flag on the slot) for the next lookup to apply. +constexpr uint32_t kTexObjBucketShift = 16; +std::unordered_map> g_texObjBuckets; std::vector g_texObjIntervalsPending; bool g_texObjIntervalsRebuildAll = true; -// Upper bound on any entry's byte size. The dirty scan walks left from lower_bound(dirtyStart) -// and needs this to know when no earlier-starting interval can still reach dirtyStart (intervals -// are sorted by start only, so a larger one could start further back). An overestimate just -// lengthens the walk, so the patch path only ever raises it and rebuild recomputes it exactly. -uint32_t g_texObjIntervalMaxSize = 0; - // Address bounds of every registered TLUT payload, so a DCStoreRange over an // unrelated buffer can skip the g_TlutObjMeta scan entirely. Recomputed lazily // because TLUT registration is far rarer than cache flushes. @@ -239,13 +238,6 @@ bool IsUsableTexObjMeta(const TexObjMeta& meta) { IsKnownTexFormat(meta.format); } -bool TexObjIntervalLess(const TexObjIntervalEntry& lhs, const TexObjIntervalEntry& rhs) { - if (lhs.dataAddr != rhs.dataAddr) { - return lhs.dataAddr < rhs.dataAddr; - } - return lhs.objAddr < rhs.objAddr; -} - bool TryBuildTexObjIntervalEntry(TexObjSlot& slot, TexObjIntervalEntry& outEntry) { if (!IsUsableTexObjMeta(slot)) { return false; @@ -298,28 +290,60 @@ TexObjSlot& FindOrCreateTexObjSlot(uint32_t addr) { // --- DCStoreRange interval index -------------------------------------------- +// The 64 KiB buckets [first, last] that bytes [start, start + size) touch (size > 0). +inline uint32_t FirstTexObjBucket(uint64_t start) { return static_cast(start >> kTexObjBucketShift); } +inline uint32_t LastTexObjBucket(uint64_t start, uint64_t size) { + return static_cast((start + size - 1u) >> kTexObjBucketShift); +} + +void IndexTexObjSlotLocked(TexObjSlot& slot, uint32_t dataAddr, uint32_t size) { + const uint32_t last = LastTexObjBucket(dataAddr, size); + for (uint64_t bucket = FirstTexObjBucket(dataAddr); bucket <= last; ++bucket) { + g_texObjBuckets[static_cast(bucket)].push_back(&slot); + } + slot.indexedDataAddr = dataAddr; + slot.indexedSize = size; + slot.inIntervalIndex = true; +} + +void UnindexTexObjSlotLocked(TexObjSlot& slot) { + if (slot.inIntervalIndex) { + const uint32_t last = LastTexObjBucket(slot.indexedDataAddr, slot.indexedSize); + for (uint64_t bucket = FirstTexObjBucket(slot.indexedDataAddr); bucket <= last; ++bucket) { + const auto it = g_texObjBuckets.find(static_cast(bucket)); + if (it == g_texObjBuckets.end()) { + continue; + } + auto& list = it->second; + const auto pos = std::find(list.begin(), list.end(), &slot); + if (pos != list.end()) { + *pos = list.back(); + list.pop_back(); + } + } + } + slot.indexedDataAddr = 0; + slot.indexedSize = 0; + slot.inIntervalIndex = false; +} + void RebuildTexObjIntervalsLocked() { - g_texObjIntervals.clear(); - g_texObjIntervals.reserve(g_TexObjMeta.size()); - g_texObjIntervalMaxSize = 0; + for (auto& [bucket, list] : g_texObjBuckets) { + (void)bucket; + list.clear(); + } for (auto& entry : g_TexObjMeta) { TexObjSlot& slot = entry.second; slot.pendingIntervalUpdate = false; + slot.inIntervalIndex = false; TexObjIntervalEntry interval{}; if (TryBuildTexObjIntervalEntry(slot, interval)) { - g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, interval.size); - g_texObjIntervals.push_back(interval); - slot.indexedDataAddr = interval.dataAddr; - slot.indexedSize = interval.size; - slot.inIntervalIndex = true; + IndexTexObjSlotLocked(slot, interval.dataAddr, interval.size); } else { slot.indexedDataAddr = 0; slot.indexedSize = 0; - slot.inIntervalIndex = false; } } - - std::sort(g_texObjIntervals.begin(), g_texObjIntervals.end(), TexObjIntervalLess); g_texObjIntervalsRebuildAll = false; g_texObjIntervalsPending.clear(); } @@ -336,94 +360,30 @@ void QueueTexObjIntervalUpdate(TexObjSlot& slot) { g_texObjIntervalsPending.push_back(&slot); } -// Returns true when the sorted index actually had to shift elements, which is -// the only expensive outcome. Metadata writebacks that leave the interval -// untouched - the overwhelming majority, since GXLoadTexObj stores its metadata -// back twice per call - cost one GXGetTexBufferSize and two compares. -bool PatchTexObjIntervalEntryLocked(TexObjSlot& slot) { +// Metadata writebacks that leave the interval untouched - the overwhelming +// majority, since GXLoadTexObj stores its metadata back twice per call - cost +// one GXGetTexBufferSize and two compares; a move touches only its buckets. +void PatchTexObjIntervalEntryLocked(TexObjSlot& slot) { TexObjIntervalEntry fresh{}; const bool hasFresh = TryBuildTexObjIntervalEntry(slot, fresh); - - if (slot.inIntervalIndex && hasFresh && slot.indexedDataAddr == fresh.dataAddr) { - if (slot.indexedSize == fresh.size) { - return false; - } - // Same sort position, different payload: fix it up in place. - const auto it = - std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), fresh, TexObjIntervalLess); - if (it != g_texObjIntervals.end() && it->dataAddr == fresh.dataAddr && it->objAddr == fresh.objAddr) { - it->size = fresh.size; - it->slot = &slot; - slot.indexedSize = fresh.size; - g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, fresh.size); - return false; - } - // Index and slot disagree; fall through and rebuild this entry. - slot.inIntervalIndex = false; + if (slot.inIntervalIndex && hasFresh && slot.indexedDataAddr == fresh.dataAddr && + slot.indexedSize == fresh.size) { + return; } - - bool moved = false; - if (slot.inIntervalIndex) { - const TexObjIntervalEntry stale{ - .dataAddr = slot.indexedDataAddr, - .size = 0, - .objAddr = slot.objAddr, - .slot = &slot, - }; - const auto staleIt = - std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), stale, TexObjIntervalLess); - if (staleIt != g_texObjIntervals.end() && staleIt->dataAddr == stale.dataAddr && - staleIt->objAddr == stale.objAddr) { - g_texObjIntervals.erase(staleIt); - moved = true; - } - slot.inIntervalIndex = false; + UnindexTexObjSlotLocked(slot); + if (hasFresh) { + IndexTexObjSlotLocked(slot, fresh.dataAddr, fresh.size); } - - slot.indexedDataAddr = 0; - slot.indexedSize = 0; - if (!hasFresh) { - return moved; - } - - const auto insertAt = - std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), fresh, TexObjIntervalLess); - g_texObjIntervals.insert(insertAt, fresh); - slot.indexedDataAddr = fresh.dataAddr; - slot.indexedSize = fresh.size; - slot.inIntervalIndex = true; - g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, fresh.size); - return true; } -// Each real move memmoves half the index on average, so a burst of genuine -// relocations (a course load re-initialising thousands of texobjs) is still -// cheaper to absorb with one rebuild. -constexpr size_t kMaxTexObjIntervalPatchMoves = 64; - void EnsureTexObjIntervalsLocked() { if (g_texObjIntervalsRebuildAll) { RebuildTexObjIntervalsLocked(); return; } - if (g_texObjIntervalsPending.empty()) { - return; - } - - size_t moves = 0; - for (size_t i = 0; i < g_texObjIntervalsPending.size(); ++i) { - TexObjSlot& slot = *g_texObjIntervalsPending[i]; - slot.pendingIntervalUpdate = false; - if (PatchTexObjIntervalEntryLocked(slot)) { - ++moves; - } - if (moves >= kMaxTexObjIntervalPatchMoves && - g_texObjIntervalsPending.size() - (i + 1) > kMaxTexObjIntervalPatchMoves) { - // RebuildTexObjIntervalsLocked clears every slot's pending flag and - // the queue itself, including the entries not visited here. - RebuildTexObjIntervalsLocked(); - return; - } + for (TexObjSlot* slot : g_texObjIntervalsPending) { + slot->pendingIntervalUpdate = false; + PatchTexObjIntervalEntryLocked(*slot); } g_texObjIntervalsPending.clear(); } @@ -649,33 +609,31 @@ void MarkTexObjsDirtyForRange(uint32_t addr, uint32_t size) { std::lock_guard guard(g_texObjMutex); EnsureTexObjIntervalsLocked(); - const auto lower = std::lower_bound( - g_texObjIntervals.begin(), g_texObjIntervals.end(), static_cast(dirtyStart), - [](const TexObjIntervalEntry& entry, uint32_t key) { return entry.dataAddr < key; }); - - for (auto it = lower; it != g_texObjIntervals.begin();) { - --it; - const uint64_t texStart = it->dataAddr; - if (texStart + g_texObjIntervalMaxSize <= dirtyStart) { - // No entry at or left of this start can reach dirtyStart anymore. - break; + // Every slot overlapping the range is in at least one of the range's buckets. + const auto markOverlapping = [&](const std::vector& list) { + for (TexObjSlot* slot : list) { + const uint64_t texStart = slot->indexedDataAddr; + if (texStart < dirtyEnd && texStart + slot->indexedSize > dirtyStart) { + slot->needsUpload = true; + } } - const uint64_t texEnd = texStart + it->size; - if (texEnd <= dirtyStart) { - // This entry falls short, but an earlier-starting, larger one may - // still overlap: keep walking until the max-size bound proves - // otherwise. - continue; + }; + const uint32_t first = FirstTexObjBucket(dirtyStart); + const uint32_t last = LastTexObjBucket(dirtyStart, size); + if (static_cast(last) - first + 1u > g_texObjBuckets.size()) { + // A range wider than the index (a whole-heap flush): visit the buckets there are instead. + for (const auto& [bucket, list] : g_texObjBuckets) { + if (bucket >= first && bucket <= last) { + markOverlapping(list); + } } - it->slot->needsUpload = true; + return; } - - for (auto it = lower; it != g_texObjIntervals.end(); ++it) { - const uint64_t texStart = it->dataAddr; - if (texStart >= dirtyEnd) { - break; + for (uint64_t bucket = first; bucket <= last; ++bucket) { + const auto it = g_texObjBuckets.find(static_cast(bucket)); + if (it != g_texObjBuckets.end()) { + markOverlapping(it->second); } - it->slot->needsUpload = true; } } From 29a9a0c207ddef606cad4bb4a3cbd614c89c43f0 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:49:34 +0100 Subject: [PATCH 11/18] bucket efb copy destinations --- runtime/src/hle/gx/gx_copy.cpp | 42 +++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/runtime/src/hle/gx/gx_copy.cpp b/runtime/src/hle/gx/gx_copy.cpp index 3dbdc8d6e..2f13b3b50 100644 --- a/runtime/src/hle/gx/gx_copy.cpp +++ b/runtime/src/hle/gx/gx_copy.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace { @@ -20,12 +21,47 @@ std::mutex g_efbCopyDestinationsMutex; std::map g_efbCopyDestinations; uint32_t g_largestEfbCopyDestination = 0; +// How many destinations touch each 64 KiB bucket of guest RAM. Nearly every DCStoreRange misses all +// destinations, and an empty bucket answers that without walking the map. +constexpr uint32_t kEfbCopyBucketShift = 16; +std::unordered_map g_efbCopyBuckets; + +void CountEfbCopyBucketsLocked(uint64_t start, uint32_t size, int32_t delta) { + const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift; + for (uint64_t bucket = start >> kEfbCopyBucketShift; bucket <= last; ++bucket) { + g_efbCopyBuckets[static_cast(bucket)] += static_cast(delta); + } +} + +bool AnyEfbCopyBucketLocked(uint64_t start, uint32_t size) { + const uint64_t first = start >> kEfbCopyBucketShift; + const uint64_t last = (start + size - 1u) >> kEfbCopyBucketShift; + if (last - first + 1u > g_efbCopyBuckets.size()) { + return true; // wider than the index; the map walk answers + } + for (uint64_t bucket = first; bucket <= last; ++bucket) { + const auto it = g_efbCopyBuckets.find(static_cast(bucket)); + if (it != g_efbCopyBuckets.end() && it->second != 0) { + return true; + } + } + return false; +} + void RememberEfbCopyDestination(uint32_t addr, uint32_t size) { if (size == 0) { return; } std::lock_guard guard(g_efbCopyDestinationsMutex); - g_efbCopyDestinations[CanonicalizeGxMainRamAddress(addr)] = size; + const uint32_t canonical = CanonicalizeGxMainRamAddress(addr); + const auto [it, inserted] = g_efbCopyDestinations.try_emplace(canonical, size); + if (inserted) { + CountEfbCopyBucketsLocked(canonical, size, +1); + } else if (it->second != size) { + CountEfbCopyBucketsLocked(canonical, it->second, -1); + CountEfbCopyBucketsLocked(canonical, size, +1); + it->second = size; + } // A conservative monotonic maximum lets invalidation jump directly to // the only map interval that could overlap instead of walking every copy // destination on every DCStoreRange. @@ -44,6 +80,9 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) { std::vector retired; { std::lock_guard guard(g_efbCopyDestinationsMutex); + if (!AnyEfbCopyBucketLocked(dirtyStart, size)) { + return; + } const uint64_t earliestCandidate = dirtyStart > g_largestEfbCopyDestination ? dirtyStart - g_largestEfbCopyDestination : 0; for (auto it = g_efbCopyDestinations.lower_bound(static_cast(earliestCandidate)); @@ -55,6 +94,7 @@ void InvalidateEfbCopyDestinationsForRange(uint32_t addr, uint32_t size) { continue; } retired.push_back(it->first); + CountEfbCopyBucketsLocked(it->first, it->second, -1); it = g_efbCopyDestinations.erase(it); } } From 60a83e793d44b3fef81dfd20a883c5c01f44134a Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:49:34 +0100 Subject: [PATCH 12/18] publish vertex state in two calls --- runtime/src/hle/gx/gx_stream_common.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/runtime/src/hle/gx/gx_stream_common.h b/runtime/src/hle/gx/gx_stream_common.h index 006ab4711..d60f4735e 100644 --- a/runtime/src/hle/gx/gx_stream_common.h +++ b/runtime/src/hle/gx/gx_stream_common.h @@ -361,19 +361,26 @@ struct AuroraVtxPublishOptions { int fmtLoopLast; }; +// One GXSetVtxDescv and one GXSetVtxAttrFmtv, which leave aurora's registers and dirty bits exactly +// as the per-attribute GXSetVtxDesc / GXSetVtxAttrFmt calls in the same order did. inline void PublishAuroraVtxState(GXVtxFmt fmt, const AuroraVtxPublishOptions& options) { + GXVtxDescList descs[27]; + size_t descCount = 0; for (int attr = 0; attr < 26; ++attr) { if (!options.includeNbt && attr == GX_VA_NBT) { continue; } - const GXAttr gxAttr = static_cast(attr); GXAttrType type = g_hleGxState.vtxDesc[attr]; if (type == GX_INDEX8 || type == GX_INDEX16) { type = GX_DIRECT; } - GXSetVtxDesc(gxAttr, type); + descs[descCount++] = GXVtxDescList{static_cast(attr), type}; } + descs[descCount] = GXVtxDescList{GX_VA_NULL, GX_NONE}; + GXSetVtxDescv(descs); + GXVtxAttrFmtList fmts[27]; + size_t fmtCount = 0; for (int attr = options.fmtLoopFirst; attr <= options.fmtLoopLast; ++attr) { if (!options.includeNbt && attr == GX_VA_NBT) { continue; @@ -383,7 +390,11 @@ inline void PublishAuroraVtxState(GXVtxFmt fmt, const AuroraVtxPublishOptions& o continue; } const auto& f = g_hleGxState.vtxAttrFmt[fmt][attr]; - GXSetVtxAttrFmt(fmt, gxAttr, f.cnt, f.type, f.frac); + fmts[fmtCount++] = GXVtxAttrFmtList{gxAttr, f.cnt, f.type, f.frac}; + } + if (fmtCount != 0) { + fmts[fmtCount] = GXVtxAttrFmtList{GX_VA_NULL, GX_POS_XYZ, GX_F32, 0}; + GXSetVtxAttrFmtv(fmt, fmts); } } From b3ff1058f6eef925ab23792116873fd34cbbfa7f Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:56:35 +0100 Subject: [PATCH 13/18] republish callback-clobberable registers after boundaries --- ...xxLinearCodeGenerator.RegisterResidency.cs | 13 ++ .../DirtyRegisterWritebackCodeGenTests.cs | 148 ++++++++++++++++-- 2 files changed, 152 insertions(+), 9 deletions(-) diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs index 4ecec87cc..a7bd0cbd0 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.RegisterResidency.cs @@ -47,6 +47,18 @@ private sealed class RegisterResidency private const byte CtrBit = 2; private const byte XerBit = 4; + // What a guest callback may leave changed in CpuContext under the PowerPC EABI (r0, r3-r12, + // f0-f13, CR, CTR, XER), plus r13, which OS_HLE_ProcessAlarms resets to the SDA base. A runtime + // native can run guest handlers on the caller's live context behind a contract that declares no + // writes (OS_HLE_ProcessAlarms from GKI_delay / PPCMtdec), so after any boundary these stay dirty + // unless it reloaded them; the callbacks preserve the rest. + private static readonly DirtySet CallbackClobberable = new() + { + Gpr = 0x00003FF9u, + Fpr = 0x00003FFFu, + Flags = CrBit | CtrBit | XerBit + }; + internal struct DirtySet : IEquatable { public uint Gpr; @@ -451,6 +463,7 @@ public void AppendReload(StringBuilder sb, string pad, ResidencyBoundarySync syn if (sync.ReloadsCr) reloaded.Flags |= CrBit; if (sync.ReloadsCtr) reloaded.Flags |= CtrBit; if (sync.ReloadsXer) reloaded.Flags |= XerBit; + NoteWrite(DirtySet.AndNot(CallbackClobberable, reloaded)); NoteClean(reloaded); if (_recording) return; foreach (var gpr in _gprs) diff --git a/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs index 4e27a8092..b490c5945 100644 --- a/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs +++ b/translator/tests/Translator.Tests/DirtyRegisterWritebackCodeGenTests.cs @@ -398,14 +398,14 @@ private static GuestAbiContract Contract(uint gprRead = 0, uint gprWrite = 0) => [Fact] public void NarrowContractFlushLeavesTheFlushedRegisterCleanWithoutAReload() { - // The callee only reads r3: the call publishes r3 and does not reload it, so the return must not publish - // it a second time. r4 (written, outside the contract) stays dirty across the call. + // The callee only reads r31: the call publishes r31 and does not reload it, so the return must not + // publish it a second time. r30 (written, outside the contract) stays dirty across the call. var function = new IrFunction("dirty_narrow", "entry", new[] { new IrBasicBlock("entry", new IrInstruction[] { - Inc("r3"), - Inc("r4"), + Inc("r31"), + Inc("r30"), new IrCall(string.Empty, "0x80002000", Array.Empty()), new IrReturn(null) }) @@ -418,15 +418,145 @@ public void NarrowContractFlushLeavesTheFlushedRegisterCleanWithoutAReload() new RepresentationEnvironment(new Dictionary()), guestAbiContracts: new Dictionary { - [0x80002000u] = Contract(gprRead: 1u << 3) + [0x80002000u] = Contract(gprRead: 1u << 31) }).ReplaceLineEndings(Lf); - Assert.Equal(1, Count(code, "ctx->gpr[3] = r3;")); - Assert.Equal(1, Count(code, "ctx->gpr[4] = r4;")); + Assert.Equal(1, Count(code, "ctx->gpr[31] = r31;")); + Assert.Equal(1, Count(code, "ctx->gpr[30] = r30;")); var call = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); Assert.True(call > 0); - Assert.DoesNotContain("ctx->gpr[3] = r3;", code[call..], StringComparison.Ordinal); - Assert.Contains("ctx->gpr[4] = r4;", code[call..], StringComparison.Ordinal); + Assert.DoesNotContain("ctx->gpr[31] = r31;", code[call..], StringComparison.Ordinal); + Assert.Contains("ctx->gpr[30] = r30;", code[call..], StringComparison.Ordinal); + } + + [Fact] + public void VolatileRegisterIsPublishedAgainAfterABoundaryThatDidNotReloadIt() + { + // The first callee only reads r3, so its flush leaves r3 equal to CpuContext. The second declares + // nothing, but a native there can run a guest handler on this context that changes r3; the return + // has to republish the resident r3, as it would without dirty tracking. + var function = new IrFunction("dirty_volatile_callback", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r3"), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(gprRead: 1u << 3), + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var second = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(second > 0); + Assert.Contains("ctx->gpr[3] = r3;", code[second..], StringComparison.Ordinal); + } + + [Fact] + public void R13ReloadedByAFullBoundaryIsPublishedAgainAfterANarrowOne() + { + // The alarm pump behind a narrow native resets CpuContext's r13 to the SDA base. + var function = new IrFunction("dirty_r13_reload", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + Inc("r13"), + new IrCall(string.Empty, "0x80004000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var narrow = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(narrow > 0); + Assert.Contains("r13 = ctx->gpr[13];", code[..narrow], StringComparison.Ordinal); + Assert.Contains("ctx->gpr[13] = r13;", code[narrow..], StringComparison.Ordinal); + } + + [Fact] + public void XerReloadedByAFullBoundaryIsPublishedAgainAfterANarrowOne() + { + var function = new IrFunction("dirty_xer_reload", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrCall("xer", "PPC_UpdateCarrySub", new[] { IrValue.Register("r3"), IrValue.Register("r4") }), + new IrCall(string.Empty, "0x80004000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + // 0x80004000 has no contract: a full boundary that reloads XER. + var narrow = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(narrow > 0); + Assert.Contains("xer = ctx->xer;", code[..narrow], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[narrow..], StringComparison.Ordinal); + } + + [Fact] + public void WrittenXerIsPublishedAgainAtEveryBoundaryThatDoesNotReloadIt() + { + // Neither callee writes XER, so neither call reloads it. A handler run behind such a contract can still + // change CpuContext's XER, which is why every boundary republishes the resident one. + var function = new IrFunction("dirty_xer", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrCall("xer", "PPC_UpdateCarrySub", new[] { IrValue.Register("r3"), IrValue.Register("r4") }), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrCall(string.Empty, "0x80003000", Array.Empty()), + new IrReturn(null) + }) + }); + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(new Dictionary()), + guestAbiContracts: new Dictionary + { + [0x80002000u] = Contract(), + [0x80003000u] = Contract() + }).ReplaceLineEndings(Lf); + + var first = code.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + var second = code.IndexOf("InvokeDirectCpu<0x80003000u>(ctx);", StringComparison.Ordinal); + Assert.True(first > 0 && second > first); + Assert.Contains("ctx->xer = xer;", code[..first], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[first..second], StringComparison.Ordinal); + Assert.Contains("ctx->xer = xer;", code[second..], StringComparison.Ordinal); } [Fact] From 7e42c1f3c64ef5fe8b8678857636a9e3d1bbde32 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:56:36 +0100 Subject: [PATCH 14/18] bump dl write generation when recording ends --- runtime/src/hle/gx/gx_frame.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/src/hle/gx/gx_frame.cpp b/runtime/src/hle/gx/gx_frame.cpp index 78fd1ccb9..d02ef06cf 100644 --- a/runtime/src/hle/gx/gx_frame.cpp +++ b/runtime/src/hle/gx/gx_frame.cpp @@ -1,4 +1,5 @@ #include "gx_internal.h" +#include "gx_guest_write.h" #include "runtime_log.h" #include @@ -26,6 +27,9 @@ void EndDisplayListRecording() { Memory::Write32(kDlCountAddr, g_dlRecordState.count); } catch (const Memory::AccessViolation&) { } + // Recording writes the list through ordinary stores, which do not move the write generation the + // display-list scan cache trusts; a list re-recorded at the same address must not reuse its old scan. + GxGuestWrite::NotifyWrite(g_dlRecordState.base, g_dlRecordState.count); } void WriteDisplayListData(uint32_t val, uint32_t sizeBytes) { From 801b80d53f94ccea9876a307b6c55f355a1fc6d9 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:56:36 +0100 Subject: [PATCH 15/18] drop dvd handle cache --- runtime/include/hle/dvd_contract.h | 69 +++++------------------------- 1 file changed, 11 insertions(+), 58 deletions(-) diff --git a/runtime/include/hle/dvd_contract.h b/runtime/include/hle/dvd_contract.h index 8f4dcf456..9cb9576a6 100644 --- a/runtime/include/hle/dvd_contract.h +++ b/runtime/include/hle/dvd_contract.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -295,57 +294,14 @@ inline bool ReadExact(const std::filesystem::path& hostPath, HostReadFailure& failure) { failure = HostReadFailure::None; - // DVD callbacks run on the guest thread, often reading the same stream - // file many times per second. Reuse a bounded set of handles without - // sharing seek positions with another host thread. - struct OpenFile { - std::filesystem::path path; - std::ifstream stream; - std::streamoff size = 0; - std::filesystem::file_time_type modified{}; - uint64_t lastUse = 0; - }; - static thread_local std::vector openFiles; - static thread_local uint64_t useCount = 0; - constexpr size_t kMaxOpenFiles = 8; - - auto found = std::find_if(openFiles.begin(), openFiles.end(), - [&](const OpenFile& entry) { return entry.path == hostPath; }); - if (found != openFiles.end()) { - std::error_code ec; - const auto modified = std::filesystem::last_write_time(hostPath, ec); - if (ec || modified != found->modified) { - openFiles.erase(found); - found = openFiles.end(); - } - } - - if (found == openFiles.end()) { - std::ifstream file(hostPath, std::ios::binary); - if (!file.is_open()) { - failure = HostReadFailure::MissingFile; - return false; - } - file.seekg(0, std::ios::end); - const std::streamoff size = file.tellg(); - if (size < 0) { - failure = HostReadFailure::BadOffset; - return false; - } - std::error_code ec; - const auto modified = std::filesystem::last_write_time(hostPath, ec); - if (openFiles.size() == kMaxOpenFiles) { - const auto oldest = std::min_element(openFiles.begin(), openFiles.end(), - [](const OpenFile& a, const OpenFile& b) { return a.lastUse < b.lastUse; }); - openFiles.erase(oldest); - } - openFiles.push_back({hostPath, std::move(file), size, - ec ? std::filesystem::file_time_type{} : modified, 0}); - found = std::prev(openFiles.end()); + std::ifstream file(hostPath, std::ios::binary); + if (!file.is_open()) { + failure = HostReadFailure::MissingFile; + return false; } - found->lastUse = ++useCount; - const std::streamoff fileSize = found->size; + file.seekg(0, std::ios::end); + const std::streamoff fileSize = file.tellg(); if (fileSize < 0 || offset > static_cast(std::numeric_limits::max()) || offset >= static_cast(fileSize)) { @@ -360,20 +316,17 @@ inline bool ReadExact(const std::filesystem::path& hostPath, } std::vector staged(length); - found->stream.clear(); - found->stream.seekg(static_cast(offset), std::ios::beg); - if (!found->stream) { + file.seekg(static_cast(offset), std::ios::beg); + if (!file) { failure = HostReadFailure::BadOffset; - openFiles.erase(found); return false; } if (length != 0) { - found->stream.read(reinterpret_cast(staged.data()), - static_cast(length)); - if (found->stream.gcount() != static_cast(length)) { + file.read(reinterpret_cast(staged.data()), + static_cast(length)); + if (file.gcount() != static_cast(length)) { failure = HostReadFailure::ShortRead; - openFiles.erase(found); return false; } } From b41d0c910a153b8e67c9a1fd06a1240715195aeb Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:22:29 +0100 Subject: [PATCH 16/18] drop register-only dl cache --- runtime/src/hle/gx/gx_dl.cpp | 7 ------- runtime/src/hle/gx/gx_frame.cpp | 4 ---- 2 files changed, 11 deletions(-) diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index a53584e65..71e18fbcd 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1333,13 +1333,6 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) const bool mayContainDraw = (cached != nullptr) ? cached->mayContainDraw : DisplayListMayContainDraw(list, nbytes); if (!mayContainDraw) { - // Register-only lists are remembered too, so the next call skips the walk. - if (cached == nullptr && allowScanCache) { - const uint64_t contentDigest = - probe.digestValid ? probe.contentDigest : DlContentDigest(list, nbytes); - StoreDlScanCache(listAddr, nbytes, scanLayoutHash, contentDigest, probe.writeGeneration, - false, DlScanCacheEntry{}, {}, nullptr, 0); - } EnsureAuroraFrameActive(); GXMarkFrameWork(); GXCallDisplayList(list, nbytes); diff --git a/runtime/src/hle/gx/gx_frame.cpp b/runtime/src/hle/gx/gx_frame.cpp index d02ef06cf..78fd1ccb9 100644 --- a/runtime/src/hle/gx/gx_frame.cpp +++ b/runtime/src/hle/gx/gx_frame.cpp @@ -1,5 +1,4 @@ #include "gx_internal.h" -#include "gx_guest_write.h" #include "runtime_log.h" #include @@ -27,9 +26,6 @@ void EndDisplayListRecording() { Memory::Write32(kDlCountAddr, g_dlRecordState.count); } catch (const Memory::AccessViolation&) { } - // Recording writes the list through ordinary stores, which do not move the write generation the - // display-list scan cache trusts; a list re-recorded at the same address must not reuse its old scan. - GxGuestWrite::NotifyWrite(g_dlRecordState.base, g_dlRecordState.count); } void WriteDisplayListData(uint32_t val, uint32_t sizeBytes) { From a36098e90706af5a313bef30d62156219a4c0ab3 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:15:58 +0100 Subject: [PATCH 17/18] drop unmeasured yaz0, dvd, texture-dirty and vertex publish ports --- runtime/src/hle/egg_decomp.cpp | 81 ++-------- runtime/src/hle/gx/gx_internal.h | 6 +- runtime/src/hle/gx/gx_objects.cpp | 218 +++++++++++++++----------- runtime/src/hle/gx/gx_stream_common.h | 17 +- runtime/src/hle/storage/dvd.cpp | 19 +-- 5 files changed, 152 insertions(+), 189 deletions(-) diff --git a/runtime/src/hle/egg_decomp.cpp b/runtime/src/hle/egg_decomp.cpp index 7fcdc2a22..94ab9ebec 100644 --- a/runtime/src/hle/egg_decomp.cpp +++ b/runtime/src/hle/egg_decomp.cpp @@ -1,19 +1,20 @@ #include "hle_stubs.h" #include -#include #include "memory.h" -#include "recomp_mod_loader.h" #include "runtime_log.h" -extern "C" void GxNotifyGuestRamDmaWrite(uint32_t addr, uint32_t size); - // Native because a crafted Yaz0 run writes past the caller's buffer // (github.com/vabold/szsHaxx) // https://github.com/vabold/Kinoko/blob/main/source/egg/core/Decomp.cc -template -static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { +extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) +{ + const uint32_t expandSize = (static_cast(MemoryInline::FlatRead8(src + 4)) << 24) | + (static_cast(MemoryInline::FlatRead8(src + 5)) << 16) | + (static_cast(MemoryInline::FlatRead8(src + 6)) << 8) | + static_cast(MemoryInline::FlatRead8(src + 7)); + uint32_t srcIdx = 16; uint32_t dstIdx = 0; uint32_t mask = 0; @@ -21,15 +22,15 @@ static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { while (static_cast(dstIdx) < static_cast(expandSize)) { if (mask == 0) { - flags = access.ReadSource(srcIdx++); + flags = MemoryInline::FlatRead8(src + srcIdx++); mask = 0x80; } if ((flags & mask) != 0) { - access.WriteOutput(dstIdx++, access.ReadSource(srcIdx++)); + MemoryInline::FlatWrite8(dst + dstIdx++, MemoryInline::FlatRead8(src + srcIdx++)); } else { - const uint32_t high = access.ReadSource(srcIdx); - const uint32_t low = access.ReadSource(srcIdx + 1); + const uint32_t high = MemoryInline::FlatRead8(src + srcIdx); + const uint32_t low = MemoryInline::FlatRead8(src + srcIdx + 1); srcIdx += 2; const uint32_t rep = (high << 8) | low; @@ -47,7 +48,7 @@ static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { uint32_t count = rep >> 12; count = count != 0 ? count + 2 - : static_cast(access.ReadSource(srcIdx++)) + 18; + : static_cast(MemoryInline::FlatRead8(src + srcIdx++)) + 18; for (uint32_t i = 0; i < count; ++i) { if (dstIdx >= expandSize) { @@ -58,7 +59,7 @@ static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { "The game stopped decoding a malformed Yaz0 file."); std::abort(); } - access.WriteOutput(dstIdx++, access.ReadOutput(copyIdx++)); + MemoryInline::FlatWrite8(dst + dstIdx++, MemoryInline::FlatRead8(dst + copyIdx++)); } } @@ -68,61 +69,5 @@ static uint32_t DecodeSZS(uint32_t src, uint32_t expandSize, Access& access) { return expandSize; } -struct GuestSZSAccess { - uint32_t src; - uint32_t dst; - - uint8_t ReadSource(uint32_t offset) const { return MemoryInline::FlatRead8(src + offset); } - uint8_t ReadOutput(uint32_t offset) const { return MemoryInline::FlatRead8(dst + offset); } - void WriteOutput(uint32_t offset, uint8_t value) const { - MemoryInline::FlatWrite8(dst + offset, value); - } -}; - -struct HostSZSAccess { - const uint8_t* src; - uint8_t* dst; - - uint8_t ReadSource(uint32_t offset) const { return src[offset]; } - uint8_t ReadOutput(uint32_t offset) const { return dst[offset]; } - void WriteOutput(uint32_t offset, uint8_t value) const { dst[offset] = value; } -}; - -static bool HasDeferredReadPages(uint32_t address, size_t length) { - const uint32_t first = address >> MemoryInline::kPageShift; - const uint32_t last = static_cast( - (static_cast(address) + length - 1) >> MemoryInline::kPageShift); - for (uint32_t page = first; page <= last; ++page) { - if (MemoryInline::g_deferredReadCoveredPages[page] != 0) return true; - } - return false; -} - -extern "C" uint32_t EGG_Decomp_decodeSZS_80218c2c(uint32_t src, uint32_t dst) -{ - const uint32_t expandSize = (static_cast(MemoryInline::FlatRead8(src + 4)) << 24) | - (static_cast(MemoryInline::FlatRead8(src + 5)) << 16) | - (static_cast(MemoryInline::FlatRead8(src + 6)) << 8) | - static_cast(MemoryInline::FlatRead8(src + 7)); - - // A token produces at least one byte; all-literal data is the largest - // valid input (one flag per eight output bytes). Three extra bytes cover - // the final malformed run before its output-overrun check aborts. - const size_t maxSourceBytes = 16ull + expandSize + (static_cast(expandSize) + 7) / 8 + 3; - if (expandSize != 0 && expandSize <= static_cast(std::numeric_limits::max()) && - !GuestFlat::RequiresCheckedAccess() && - Memory::Contains(src, maxSourceBytes) && Memory::Contains(dst, expandSize) && - !HasDeferredReadPages(src, maxSourceBytes) && !HasDeferredReadPages(dst, expandSize) && - !RecompMod::ExecutableWriteGuardMayHit(dst, expandSize)) { - HostSZSAccess access{Memory::GetPointer(src, maxSourceBytes), Memory::GetPointer(dst, expandSize)}; - const uint32_t decoded = DecodeSZS(src, expandSize, access); - GxNotifyGuestRamDmaWrite(dst, decoded); - return decoded; - } - - GuestSZSAccess access{src, dst}; - return DecodeSZS(src, expandSize, access); -} - PPC_NATIVE_OVERRIDE(80218C2C, EGG_Decomp_decodeSZS_80218c2c, uint32_t, (uint32_t src, uint32_t dst), (src, dst)); diff --git a/runtime/src/hle/gx/gx_internal.h b/runtime/src/hle/gx/gx_internal.h index 98dcc3a38..b8270a9b9 100644 --- a/runtime/src/hle/gx/gx_internal.h +++ b/runtime/src/hle/gx/gx_internal.h @@ -154,9 +154,9 @@ struct TexObjSlot : TexObjMeta { uint64_t guestShadow[4] = {0, 0, 0, 0}; bool hasGuestShadow = false; - // DCStoreRange interval-index bookkeeping: the range this slot is listed - // under in g_texObjBuckets, so a pending update that turns out to be a - // no-op costs a compare instead of an unlist + relist. + // DCStoreRange interval-index bookkeeping: what this slot currently + // contributes to g_texObjIntervals, so a pending update that turns out to + // be a no-op costs a compare instead of an erase + insert. uint32_t indexedDataAddr = 0; uint32_t indexedSize = 0; bool inIntervalIndex = false; diff --git a/runtime/src/hle/gx/gx_objects.cpp b/runtime/src/hle/gx/gx_objects.cpp index 7e1f554ed..645658dfb 100644 --- a/runtime/src/hle/gx/gx_objects.cpp +++ b/runtime/src/hle/gx/gx_objects.cpp @@ -1,11 +1,8 @@ #include "gx_internal.h" #include "runtime_log.h" -#include #include #include -#include -#include // --- Texture and TLUT Objects --- @@ -87,17 +84,21 @@ struct TexObjIntervalEntry { TexObjSlot* slot = nullptr; }; -// DCStoreRange invalidation index: every indexed slot is listed in each 64 KiB bucket of guest RAM -// its texture data overlaps (slot.indexedDataAddr/indexedSize), so a flushed range only looks at -// the slots in its own buckets. It replaced a vector sorted by data address, which shifted the whole -// index whenever a texture object moved (memmove was 0.7% of the game thread on a slow Android device, in -// courses), and walked back over every entry within the largest texture's size of each flush. -// Mutations queue their slot (an O(1) dedupe flag on the slot) for the next lookup to apply. -constexpr uint32_t kTexObjBucketShift = 16; -std::unordered_map> g_texObjBuckets; +// DCStoreRange invalidation index, sorted by (dataAddr, objAddr). Rebuilding is O(N log N) plus a +// GXGetTexBufferSize per texobj, so it must not happen once per DCStoreRange: mutations instead +// queue their slot for a targeted patch. The queue used to be a capacity-limited vector that +// overflowed under real per-frame GXLoadTexObj volume and silently degraded into a full rebuild +// every call; the dedupe flag now lives on the slot so queuing is O(1) and uncapped. +std::vector g_texObjIntervals; std::vector g_texObjIntervalsPending; bool g_texObjIntervalsRebuildAll = true; +// Upper bound on any entry's byte size. The dirty scan walks left from lower_bound(dirtyStart) +// and needs this to know when no earlier-starting interval can still reach dirtyStart (intervals +// are sorted by start only, so a larger one could start further back). An overestimate just +// lengthens the walk, so the patch path only ever raises it and rebuild recomputes it exactly. +uint32_t g_texObjIntervalMaxSize = 0; + // Address bounds of every registered TLUT payload, so a DCStoreRange over an // unrelated buffer can skip the g_TlutObjMeta scan entirely. Recomputed lazily // because TLUT registration is far rarer than cache flushes. @@ -238,6 +239,13 @@ bool IsUsableTexObjMeta(const TexObjMeta& meta) { IsKnownTexFormat(meta.format); } +bool TexObjIntervalLess(const TexObjIntervalEntry& lhs, const TexObjIntervalEntry& rhs) { + if (lhs.dataAddr != rhs.dataAddr) { + return lhs.dataAddr < rhs.dataAddr; + } + return lhs.objAddr < rhs.objAddr; +} + bool TryBuildTexObjIntervalEntry(TexObjSlot& slot, TexObjIntervalEntry& outEntry) { if (!IsUsableTexObjMeta(slot)) { return false; @@ -290,60 +298,28 @@ TexObjSlot& FindOrCreateTexObjSlot(uint32_t addr) { // --- DCStoreRange interval index -------------------------------------------- -// The 64 KiB buckets [first, last] that bytes [start, start + size) touch (size > 0). -inline uint32_t FirstTexObjBucket(uint64_t start) { return static_cast(start >> kTexObjBucketShift); } -inline uint32_t LastTexObjBucket(uint64_t start, uint64_t size) { - return static_cast((start + size - 1u) >> kTexObjBucketShift); -} - -void IndexTexObjSlotLocked(TexObjSlot& slot, uint32_t dataAddr, uint32_t size) { - const uint32_t last = LastTexObjBucket(dataAddr, size); - for (uint64_t bucket = FirstTexObjBucket(dataAddr); bucket <= last; ++bucket) { - g_texObjBuckets[static_cast(bucket)].push_back(&slot); - } - slot.indexedDataAddr = dataAddr; - slot.indexedSize = size; - slot.inIntervalIndex = true; -} - -void UnindexTexObjSlotLocked(TexObjSlot& slot) { - if (slot.inIntervalIndex) { - const uint32_t last = LastTexObjBucket(slot.indexedDataAddr, slot.indexedSize); - for (uint64_t bucket = FirstTexObjBucket(slot.indexedDataAddr); bucket <= last; ++bucket) { - const auto it = g_texObjBuckets.find(static_cast(bucket)); - if (it == g_texObjBuckets.end()) { - continue; - } - auto& list = it->second; - const auto pos = std::find(list.begin(), list.end(), &slot); - if (pos != list.end()) { - *pos = list.back(); - list.pop_back(); - } - } - } - slot.indexedDataAddr = 0; - slot.indexedSize = 0; - slot.inIntervalIndex = false; -} - void RebuildTexObjIntervalsLocked() { - for (auto& [bucket, list] : g_texObjBuckets) { - (void)bucket; - list.clear(); - } + g_texObjIntervals.clear(); + g_texObjIntervals.reserve(g_TexObjMeta.size()); + g_texObjIntervalMaxSize = 0; for (auto& entry : g_TexObjMeta) { TexObjSlot& slot = entry.second; slot.pendingIntervalUpdate = false; - slot.inIntervalIndex = false; TexObjIntervalEntry interval{}; if (TryBuildTexObjIntervalEntry(slot, interval)) { - IndexTexObjSlotLocked(slot, interval.dataAddr, interval.size); + g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, interval.size); + g_texObjIntervals.push_back(interval); + slot.indexedDataAddr = interval.dataAddr; + slot.indexedSize = interval.size; + slot.inIntervalIndex = true; } else { slot.indexedDataAddr = 0; slot.indexedSize = 0; + slot.inIntervalIndex = false; } } + + std::sort(g_texObjIntervals.begin(), g_texObjIntervals.end(), TexObjIntervalLess); g_texObjIntervalsRebuildAll = false; g_texObjIntervalsPending.clear(); } @@ -360,30 +336,94 @@ void QueueTexObjIntervalUpdate(TexObjSlot& slot) { g_texObjIntervalsPending.push_back(&slot); } -// Metadata writebacks that leave the interval untouched - the overwhelming -// majority, since GXLoadTexObj stores its metadata back twice per call - cost -// one GXGetTexBufferSize and two compares; a move touches only its buckets. -void PatchTexObjIntervalEntryLocked(TexObjSlot& slot) { +// Returns true when the sorted index actually had to shift elements, which is +// the only expensive outcome. Metadata writebacks that leave the interval +// untouched - the overwhelming majority, since GXLoadTexObj stores its metadata +// back twice per call - cost one GXGetTexBufferSize and two compares. +bool PatchTexObjIntervalEntryLocked(TexObjSlot& slot) { TexObjIntervalEntry fresh{}; const bool hasFresh = TryBuildTexObjIntervalEntry(slot, fresh); - if (slot.inIntervalIndex && hasFresh && slot.indexedDataAddr == fresh.dataAddr && - slot.indexedSize == fresh.size) { - return; + + if (slot.inIntervalIndex && hasFresh && slot.indexedDataAddr == fresh.dataAddr) { + if (slot.indexedSize == fresh.size) { + return false; + } + // Same sort position, different payload: fix it up in place. + const auto it = + std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), fresh, TexObjIntervalLess); + if (it != g_texObjIntervals.end() && it->dataAddr == fresh.dataAddr && it->objAddr == fresh.objAddr) { + it->size = fresh.size; + it->slot = &slot; + slot.indexedSize = fresh.size; + g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, fresh.size); + return false; + } + // Index and slot disagree; fall through and rebuild this entry. + slot.inIntervalIndex = false; } - UnindexTexObjSlotLocked(slot); - if (hasFresh) { - IndexTexObjSlotLocked(slot, fresh.dataAddr, fresh.size); + + bool moved = false; + if (slot.inIntervalIndex) { + const TexObjIntervalEntry stale{ + .dataAddr = slot.indexedDataAddr, + .size = 0, + .objAddr = slot.objAddr, + .slot = &slot, + }; + const auto staleIt = + std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), stale, TexObjIntervalLess); + if (staleIt != g_texObjIntervals.end() && staleIt->dataAddr == stale.dataAddr && + staleIt->objAddr == stale.objAddr) { + g_texObjIntervals.erase(staleIt); + moved = true; + } + slot.inIntervalIndex = false; } + + slot.indexedDataAddr = 0; + slot.indexedSize = 0; + if (!hasFresh) { + return moved; + } + + const auto insertAt = + std::lower_bound(g_texObjIntervals.begin(), g_texObjIntervals.end(), fresh, TexObjIntervalLess); + g_texObjIntervals.insert(insertAt, fresh); + slot.indexedDataAddr = fresh.dataAddr; + slot.indexedSize = fresh.size; + slot.inIntervalIndex = true; + g_texObjIntervalMaxSize = std::max(g_texObjIntervalMaxSize, fresh.size); + return true; } +// Each real move memmoves half the index on average, so a burst of genuine +// relocations (a course load re-initialising thousands of texobjs) is still +// cheaper to absorb with one rebuild. +constexpr size_t kMaxTexObjIntervalPatchMoves = 64; + void EnsureTexObjIntervalsLocked() { if (g_texObjIntervalsRebuildAll) { RebuildTexObjIntervalsLocked(); return; } - for (TexObjSlot* slot : g_texObjIntervalsPending) { - slot->pendingIntervalUpdate = false; - PatchTexObjIntervalEntryLocked(*slot); + if (g_texObjIntervalsPending.empty()) { + return; + } + + size_t moves = 0; + for (size_t i = 0; i < g_texObjIntervalsPending.size(); ++i) { + TexObjSlot& slot = *g_texObjIntervalsPending[i]; + slot.pendingIntervalUpdate = false; + if (PatchTexObjIntervalEntryLocked(slot)) { + ++moves; + } + if (moves >= kMaxTexObjIntervalPatchMoves && + g_texObjIntervalsPending.size() - (i + 1) > kMaxTexObjIntervalPatchMoves) { + // RebuildTexObjIntervalsLocked clears every slot's pending flag and + // the queue itself, including the entries not visited here. + RebuildTexObjIntervalsLocked(); + return; + } } g_texObjIntervalsPending.clear(); } @@ -609,31 +649,33 @@ void MarkTexObjsDirtyForRange(uint32_t addr, uint32_t size) { std::lock_guard guard(g_texObjMutex); EnsureTexObjIntervalsLocked(); - // Every slot overlapping the range is in at least one of the range's buckets. - const auto markOverlapping = [&](const std::vector& list) { - for (TexObjSlot* slot : list) { - const uint64_t texStart = slot->indexedDataAddr; - if (texStart < dirtyEnd && texStart + slot->indexedSize > dirtyStart) { - slot->needsUpload = true; - } + const auto lower = std::lower_bound( + g_texObjIntervals.begin(), g_texObjIntervals.end(), static_cast(dirtyStart), + [](const TexObjIntervalEntry& entry, uint32_t key) { return entry.dataAddr < key; }); + + for (auto it = lower; it != g_texObjIntervals.begin();) { + --it; + const uint64_t texStart = it->dataAddr; + if (texStart + g_texObjIntervalMaxSize <= dirtyStart) { + // No entry at or left of this start can reach dirtyStart anymore. + break; } - }; - const uint32_t first = FirstTexObjBucket(dirtyStart); - const uint32_t last = LastTexObjBucket(dirtyStart, size); - if (static_cast(last) - first + 1u > g_texObjBuckets.size()) { - // A range wider than the index (a whole-heap flush): visit the buckets there are instead. - for (const auto& [bucket, list] : g_texObjBuckets) { - if (bucket >= first && bucket <= last) { - markOverlapping(list); - } + const uint64_t texEnd = texStart + it->size; + if (texEnd <= dirtyStart) { + // This entry falls short, but an earlier-starting, larger one may + // still overlap: keep walking until the max-size bound proves + // otherwise. + continue; } - return; + it->slot->needsUpload = true; } - for (uint64_t bucket = first; bucket <= last; ++bucket) { - const auto it = g_texObjBuckets.find(static_cast(bucket)); - if (it != g_texObjBuckets.end()) { - markOverlapping(it->second); + + for (auto it = lower; it != g_texObjIntervals.end(); ++it) { + const uint64_t texStart = it->dataAddr; + if (texStart >= dirtyEnd) { + break; } + it->slot->needsUpload = true; } } diff --git a/runtime/src/hle/gx/gx_stream_common.h b/runtime/src/hle/gx/gx_stream_common.h index d60f4735e..006ab4711 100644 --- a/runtime/src/hle/gx/gx_stream_common.h +++ b/runtime/src/hle/gx/gx_stream_common.h @@ -361,26 +361,19 @@ struct AuroraVtxPublishOptions { int fmtLoopLast; }; -// One GXSetVtxDescv and one GXSetVtxAttrFmtv, which leave aurora's registers and dirty bits exactly -// as the per-attribute GXSetVtxDesc / GXSetVtxAttrFmt calls in the same order did. inline void PublishAuroraVtxState(GXVtxFmt fmt, const AuroraVtxPublishOptions& options) { - GXVtxDescList descs[27]; - size_t descCount = 0; for (int attr = 0; attr < 26; ++attr) { if (!options.includeNbt && attr == GX_VA_NBT) { continue; } + const GXAttr gxAttr = static_cast(attr); GXAttrType type = g_hleGxState.vtxDesc[attr]; if (type == GX_INDEX8 || type == GX_INDEX16) { type = GX_DIRECT; } - descs[descCount++] = GXVtxDescList{static_cast(attr), type}; + GXSetVtxDesc(gxAttr, type); } - descs[descCount] = GXVtxDescList{GX_VA_NULL, GX_NONE}; - GXSetVtxDescv(descs); - GXVtxAttrFmtList fmts[27]; - size_t fmtCount = 0; for (int attr = options.fmtLoopFirst; attr <= options.fmtLoopLast; ++attr) { if (!options.includeNbt && attr == GX_VA_NBT) { continue; @@ -390,11 +383,7 @@ inline void PublishAuroraVtxState(GXVtxFmt fmt, const AuroraVtxPublishOptions& o continue; } const auto& f = g_hleGxState.vtxAttrFmt[fmt][attr]; - fmts[fmtCount++] = GXVtxAttrFmtList{gxAttr, f.cnt, f.type, f.frac}; - } - if (fmtCount != 0) { - fmts[fmtCount] = GXVtxAttrFmtList{GX_VA_NULL, GX_POS_XYZ, GX_F32, 0}; - GXSetVtxAttrFmtv(fmt, fmts); + GXSetVtxAttrFmt(fmt, gxAttr, f.cnt, f.type, f.frac); } } diff --git a/runtime/src/hle/storage/dvd.cpp b/runtime/src/hle/storage/dvd.cpp index 2c55c46eb..b76e5e19c 100644 --- a/runtime/src/hle/storage/dvd.cpp +++ b/runtime/src/hle/storage/dvd.cpp @@ -103,23 +103,10 @@ extern "C" void DVDInit_8015EA1C(); extern "C" uint32_t g_dvdFstReservedBase; extern "C" uint32_t g_dvdFstReservedSize; -// Copy into guest RAM plus the DMA notification the GX caches need. A fully writable 1 MiB page -// takes its bytes in one memcpy, since Write8 is a plain store there; any other page keeps Write8. +// Byte-wise copy into guest RAM plus the DMA notification the GX caches need. static void CopyToGuestAsDma(uint32_t dest, const uint8_t* data, size_t size) { - size_t done = 0; - while (done < size) { - const uint32_t address = dest + static_cast(done); - const size_t pageLeft = MemoryInline::kPageSize - (address & MemoryInline::kPageMask); - const size_t chunk = std::min(size - done, pageLeft); - if (const uintptr_t bias = MemoryInline::g_fullWritablePageBias[address >> MemoryInline::kPageShift]; - bias != 0) { - std::memcpy(reinterpret_cast((bias - 1u) + address), data + done, chunk); - } else { - for (size_t i = 0; i < chunk; ++i) { - Memory::Write8(address + static_cast(i), data[done + i]); - } - } - done += chunk; + for (size_t i = 0; i < size; ++i) { + Memory::Write8(dest + static_cast(i), data[i]); } GxNotifyGuestRamDmaWrite(dest, static_cast(size)); } From 23c035a4012458f0ddea0e980e689f17240aad8c Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:33:42 +0100 Subject: [PATCH 18/18] bulk copy dvd reads into writable pages --- runtime/src/hle/storage/dvd.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/runtime/src/hle/storage/dvd.cpp b/runtime/src/hle/storage/dvd.cpp index b76e5e19c..2c55c46eb 100644 --- a/runtime/src/hle/storage/dvd.cpp +++ b/runtime/src/hle/storage/dvd.cpp @@ -103,10 +103,23 @@ extern "C" void DVDInit_8015EA1C(); extern "C" uint32_t g_dvdFstReservedBase; extern "C" uint32_t g_dvdFstReservedSize; -// Byte-wise copy into guest RAM plus the DMA notification the GX caches need. +// Copy into guest RAM plus the DMA notification the GX caches need. A fully writable 1 MiB page +// takes its bytes in one memcpy, since Write8 is a plain store there; any other page keeps Write8. static void CopyToGuestAsDma(uint32_t dest, const uint8_t* data, size_t size) { - for (size_t i = 0; i < size; ++i) { - Memory::Write8(dest + static_cast(i), data[i]); + size_t done = 0; + while (done < size) { + const uint32_t address = dest + static_cast(done); + const size_t pageLeft = MemoryInline::kPageSize - (address & MemoryInline::kPageMask); + const size_t chunk = std::min(size - done, pageLeft); + if (const uintptr_t bias = MemoryInline::g_fullWritablePageBias[address >> MemoryInline::kPageShift]; + bias != 0) { + std::memcpy(reinterpret_cast((bias - 1u) + address), data + done, chunk); + } else { + for (size_t i = 0; i < chunk; ++i) { + Memory::Write8(address + static_cast(i), data[done + i]); + } + } + done += chunk; } GxNotifyGuestRamDmaWrite(dest, static_cast(size)); }