From 93a6c53b8eac985f9db627273b7277a9ededc880 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:48:14 +0100 Subject: [PATCH 1/4] fix lr continuation resume labels --- .../CodeGen/CxxLinearCodeGenerator.cs | 61 ++++++++++++++++--- .../KamekLrContinuationIntegrationTests.cs | 10 ++- .../LrContinuationCodeGenTests.cs | 52 ++++++++++++++++ 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs index ed6d82c0f..6c37b2781 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs @@ -132,10 +132,15 @@ public CxxEmissionResult EmitWithFacts(uint entryPoint, gqrConstantsRequireRuntimeGuard); // Call-free proven kernels amortize very small ranges. Ordinary functions // use the epoch-safe scalar form and a higher threshold to avoid adding - // pointer locals for incidental field pairs. - func = GuestMemoryRangeLowering.Lower( - func, minimumAccesses: hasProvenKernelEntry ? 2 : 8, ignoreDisabledPcTraces: true, - scalarOnly: !hasProvenKernelEntry); + // pointer locals for incidental field pairs. Range resolves and access pairs + // are placed by the static CFG, so a body that can be resumed at any + // instruction label keeps its plain accesses. + if (!HasOutOfBandLocalEntry(func, nonReturningCallTargets, lrContinuationCallTargets)) + { + func = GuestMemoryRangeLowering.Lower( + func, minimumAccesses: hasProvenKernelEntry ? 2 : 8, ignoreDisabledPcTraces: true, + scalarOnly: !hasProvenKernelEntry); + } var architecturalFunc = func; var guestAbiContract = GuestAbiContractAnalyzer.Analyze(architecturalFunc, guestAbiContracts); var cfg = IrCfg.Build(func); @@ -377,9 +382,18 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool var blockSuppressed = suppressedInstructionMasks.TryGetValue(block.Label, out var suppressedMask) ? suppressedMask : null; + var continuationLabelsByStart = MapContinuationLabelsToInstructionStarts( + block.Instructions, instructionContinuationLabels); for (var i = 0; i < nonTerminatorCount; i++) { + if (continuationLabelsByStart is not null && + continuationLabelsByStart.TryGetValue(i, out var continuationLabel)) + { + RecordEmittedLocalLabel(continuationLabel); + body.AppendLine($"{continuationLabel}: ;"); + } + var directCallOrdinal = -1; if (block.Instructions[i] is IrCall ordinalCall) { @@ -397,12 +411,6 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool continue; } - if (block.Instructions[i] is IrTracePpc trace && - instructionContinuationLabels.TryGetValue(trace.Address, out var continuationLabel)) - { - RecordEmittedLocalLabel(continuationLabel); - body.AppendLine($"{continuationLabel}: ;"); - } var localFallthroughLr = TryGetLocalFallthroughLr(block.Instructions, i, nonReturningCallTargets, lrContinuationCallTargets); // State-free bodies are cloned after register caching and lose their // direct-call fallbacks, so this initial emission (which must stay a @@ -712,6 +720,39 @@ private static string EscapeCxxStringLiteral(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal) .Replace("\"", "\\\"", StringComparison.Ordinal); + /// + /// A guest instruction's trace follows its lowered IR, but a resume through LR + /// must run that instruction, so its label goes where the instruction's IR starts. + /// + private static Dictionary? MapContinuationLabelsToInstructionStarts( + IReadOnlyList instructions, + IReadOnlyDictionary continuationLabels) + { + if (continuationLabels.Count == 0) + { + return null; + } + + Dictionary? labelsByStart = null; + var start = 0; + for (var i = 0; i < instructions.Count; i++) + { + if (instructions[i] is not IrTracePpc trace) + { + continue; + } + + if (continuationLabels.TryGetValue(trace.Address, out var label)) + { + (labelsByStart ??= new Dictionary())[start] = label; + } + + start = i + 1; + } + + return labelsByStart; + } + private static uint? TryGetLocalFallthroughLr( IReadOnlyList instructions, int index, diff --git a/translator/tests/Translator.Tests/KamekLrContinuationIntegrationTests.cs b/translator/tests/Translator.Tests/KamekLrContinuationIntegrationTests.cs index 689136ba3..cbcc6502e 100644 --- a/translator/tests/Translator.Tests/KamekLrContinuationIntegrationTests.cs +++ b/translator/tests/Translator.Tests/KamekLrContinuationIntegrationTests.cs @@ -129,7 +129,12 @@ private static void AssertResumeDispatch(string source, uint callSite, uint cont Assert.True(guard > call, $"The hook must dispatch its adjusted return address.\n{source}"); Assert.Contains($"case 0x{continuation:X8}u:", source[guard..]); Assert.Contains($"goto loc_{continuation:X8};", source[guard..]); - Assert.Contains($"loc_{continuation:X8}:", source); + var label = source.IndexOf($"loc_{continuation:X8}: ;", StringComparison.Ordinal); + Assert.True(label >= 0, source); + // Resuming must execute the instruction at the continuation address, not skip it. + var next = source.IndexOf("loc_", label + 1, StringComparison.Ordinal); + var resumed = next < 0 ? source[label..] : source[label..next]; + Assert.Contains("r3 = (r3 + 2);", resumed); } private static uint[] SkipReturn(int offset) => @@ -151,8 +156,9 @@ private static TranslationSourceBundle Translate(int tailDepth, uint[] body, Directory.CreateDirectory(root); try { + // Caller + 24 is the resume address, so it holds a real instruction (addi r3,r3,2). uint[] caller = [0x60000000u, 0x38630001u, 0x38630001u, 0x38630001u, - 0x38630001u, 0x38630001u, 0x60000000u, 0x4E800020u]; + 0x38630001u, 0x38630001u, 0x38630002u, 0x4E800020u]; var hasSecondCaller = sharedTarget || companionBody is not null; var baseWords = Enumerable.Repeat(0x4E800020u, hasSecondCaller ? 24 : 8).ToArray(); caller.CopyTo(baseWords, 0); diff --git a/translator/tests/Translator.Tests/LrContinuationCodeGenTests.cs b/translator/tests/Translator.Tests/LrContinuationCodeGenTests.cs index c11bff68a..dffa9600f 100644 --- a/translator/tests/Translator.Tests/LrContinuationCodeGenTests.cs +++ b/translator/tests/Translator.Tests/LrContinuationCodeGenTests.cs @@ -60,4 +60,56 @@ public void CodeGenerator_DispatchesGuestCallLrContinuationWithoutMarkingTargetN Assert.True(fallthroughAssignmentIndex > returnIndex, code); Assert.Contains("goto loc_800E5934;", code); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CodeGenerator_KeepsPlainAccessesInResumableBodies(bool resumable) + { + const uint entry = 0x80001000u; + var instructions = new List + { + new IrAssign("lr", IrValue.Imm(unchecked((int)(entry + 4)))), + new IrCall(string.Empty, "0x8179AC3C", Array.Empty()), + new IrTracePpc(entry, "bl", "0x48000001") + }; + var registers = new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32 + }; + for (var i = 0; i < 8; i++) + { + var destination = $"r{5 + i}"; + registers[destination] = ValueRepresentation.UInt32; + instructions.Add(new IrLoad(destination, new IrAddress("r4", i * 4), 4)); + instructions.Add(new IrTracePpc(entry + 4 + (uint)i * 4, "lwz", "0x80000000")); + } + instructions.Add(new IrReturn(null)); + var function = new IrFunction("resumable_loads", $"0x{entry:X8}", + new[] { new IrBasicBlock($"0x{entry:X8}", instructions) }); + + var code = new CxxLinearCodeGenerator().Emit( + entry, + new SsaTransformer().Convert(function), + new FunctionAbiClassification("resumable_loads", ValueRepresentation.Void), + new RepresentationEnvironment(registers), + lrContinuationCallTargets: resumable ? new HashSet { 0x8179AC3Cu } : new HashSet()); + + if (!resumable) + { + // Control: the same body is range-lowered and paired when nothing can resume it mid-block. + Assert.Contains("ReadResolvedPair32", code); + return; + } + + // A resume past the range resolve would read through a missing or stale host pointer. + Assert.DoesNotContain("guest_range_", code); + // Resuming at the second load must run that load and not the first one. + var label = code.IndexOf($"loc_{entry + 8:X8}: ;", StringComparison.Ordinal); + Assert.True(label >= 0, code); + var resumed = code[label..]; + Assert.DoesNotContain("r5 = ", resumed); + Assert.Contains("r6 = ", resumed); + } } From 887e92ee3a7a20e7a0eff89f3f31fe82d1eb50f7 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:01:41 +0100 Subject: [PATCH 2/4] keep fifo stores in resumable bodies --- .../CodeGen/CxxLinearCodeGenerator.cs | 6 ++- .../GpuFifoBurstCodeGenTests.cs | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs index 6c37b2781..e67488403 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs @@ -275,8 +275,10 @@ block.Instructions[index] is not IrCall call || var nextBlockLabels = BuildNextBlockLabelMap(func); var knownConstants = BuildKnownUIntConstants(func); // Burst coalescing is planned once against the final IR so every - // emission of this body sees the same runs and the same buffer names. - var gpuFifoBurstPlan = BuildGpuFifoBurstPlan(func, knownConstants, enableGpuFifoBurstCoalescing); + // emission of this body sees the same runs and the same buffer names. A + // resume into the middle of a burst would flush slots it never wrote. + var gpuFifoBurstPlan = BuildGpuFifoBurstPlan( + func, knownConstants, enableGpuFifoBurstCoalescing && !needsInstructionContinuationLabels); // The state-free clone is derived by rewriting this body's text, and it // rewrites the ctx-carrying PSQ helper spellings into the explicit-state // ones. Hoisting changes those spellings, so leave it off for the bodies diff --git a/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs b/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs index b793ccbf2..0369e7d52 100644 --- a/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs +++ b/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs @@ -250,6 +250,46 @@ public void UnsupportedFloatWidthFallsBackToGuestMemory() Assert.Contains("MemoryInline::FlatWriteFloat64", code, StringComparison.Ordinal); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ResumableBodiesKeepIndividualFifoStores(bool resumable) + { + // A hook resuming at a later store would otherwise flush burst slots it never wrote. + var body = new List + { + new IrAssign("lr", IrValue.Imm(0x80001008)), + new IrCall(string.Empty, "0x8179AC3C", Array.Empty()), + new IrTracePpc(0x80001004u, "bl", "0x48000001") + }; + for (var index = 0; index < 4; index++) + { + body.Add(FifoStore("r3", 4)); + body.Add(new IrTracePpc(0x80001008u + (uint)index * 4, "stw", "0x90000000")); + } + body.Add(new IrReturn(null)); + var function = Function("burst_resumable", body.ToArray()); + var registers = Enumerable.Range(0, 12) + .ToDictionary(index => $"r{index}", _ => ValueRepresentation.UInt32); + registers["lr"] = ValueRepresentation.UInt32; + + var code = new CxxLinearCodeGenerator().Emit( + 0x80001000u, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(registers), + lrContinuationCallTargets: resumable ? new HashSet { 0x8179AC3Cu } : new HashSet()); + + if (!resumable) + { + Assert.Contains("GX_HLE_FIFO_WriteBurst(", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("GX_HLE_FIFO_WriteBurst(", code, StringComparison.Ordinal); + Assert.Equal(4, CountOccurrences(code, "GX_HLE_FIFO_Write32(")); + } + private static int CountOccurrences(string text, string value) { var count = 0; From 56eff21d671fedf9a17826197e708f6a7737d7db Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:05:53 +0100 Subject: [PATCH 3/4] keep resumable bodies sound across resume labels --- .../Analysis/StackAddressFacts.cs | 19 +- .../CxxLinearCodeGenerator.EmitInstruction.cs | 50 +- .../CxxLinearCodeGenerator.Expressions.cs | 8 +- .../CodeGen/CxxLinearCodeGenerator.Locals.cs | 332 ++++++------ .../CodeGen/CxxLinearCodeGenerator.cs | 201 ++++++-- .../GpuFifoBurstCodeGenTests.cs | 9 +- .../ResumableBodyCodeGenTests.cs | 472 ++++++++++++++++++ 7 files changed, 900 insertions(+), 191 deletions(-) create mode 100644 translator/tests/Translator.Tests/ResumableBodyCodeGenTests.cs diff --git a/translator/src/Translator.Core/Analysis/StackAddressFacts.cs b/translator/src/Translator.Core/Analysis/StackAddressFacts.cs index 44b709ae5..9577546ef 100644 --- a/translator/src/Translator.Core/Analysis/StackAddressFacts.cs +++ b/translator/src/Translator.Core/Analysis/StackAddressFacts.cs @@ -21,7 +21,13 @@ private StackAddressFacts(Dictionary temporaryOffsets) public static StackAddressFacts Empty { get; } = new(new Dictionary(StringComparer.OrdinalIgnoreCase)); - public static StackAddressFacts Build(IrFunction function) + /// + /// False for a body that can be resumed at any instruction: a copy into a + /// guest GPR (mr r4,r1) belongs to an earlier instruction a resume can skip. + /// Lowering temporaries live inside one instruction and r1 stays the stack + /// pointer, so those still resolve. + /// + public static StackAddressFacts Build(IrFunction function, bool trustRegisterCopies = true) { var temporaryOffsets = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var block in function.Blocks) @@ -33,11 +39,13 @@ public static StackAddressFacts Build(IrFunction function) // SSA form makes an architectural register destination as stable as a compiler // temporary (e.g. PPC prologues copying r1 to r11 for the save area). case IrAssign assign when + (trustRegisterCopies || !IsGuestGpr(assign.Destination)) && TryResolveValue(assign.Value, temporaryOffsets, out var assignOffset): temporaryOffsets[assign.Destination] = assignOffset; break; case IrBinary binary when + (trustRegisterCopies || !IsGuestGpr(binary.Destination)) && TryResolveBinary(binary, temporaryOffsets, out var binaryOffset): temporaryOffsets[binary.Destination] = binaryOffset; break; @@ -169,6 +177,15 @@ private static bool TryApplyOffset(string op, int baseOffset, int constant, out } } + private static bool IsGuestGpr(string name) + { + var baseName = GetRegisterBaseName(name); + return baseName.Length > 1 && + (baseName[0] == 'r' || baseName[0] == 'R') && + int.TryParse(baseName.AsSpan(1), out var index) && + index is >= 0 and < 32; + } + private static string GetRegisterBaseName(string name) { var underscore = name.IndexOf('_'); diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs index 8c35b11a1..86285c63b 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs @@ -28,13 +28,17 @@ private static void EmitInstruction(string currentLabel, int directCallOrdinal, IReadOnlyDictionary stateFreeCallSymbols, IReadOnlyDictionary stateFreeCallSiteVariants, IReadOnlySet modOverridableCallTargets, - bool shareLrContinuationDispatch) + bool shareLrContinuationDispatch, + IReadOnlyDictionary>? resumeLabelPairedStates) { if (ins is IrPhi) { return; // Phi nodes are eliminated before codegen. } + // Only a body that a callee can resume mid-block has resume targets. + var resumableBody = resumeLabelPairedStates is not null; + var pad = IndentPad(indent); switch (ins) { @@ -125,14 +129,18 @@ private static void EmitInstruction(string currentLabel, int directCallOrdinal, srcExpr = $"PPC_PsToScalarInline({srcExpr})"; } - if (TryEmitKnownGpuFifoStore(store, srcExpr, isFloatStore, sb, pad, knownConstants)) + // Both forms below trust the constant a static def produced: the + // FIFO form drops the address and the RAM form skips the MMIO test. + // A resumed body may not have run that def. The checked MMIO load + // above still reads the runtime address, so it stays. + if (!resumableBody && TryEmitKnownGpuFifoStore(store, srcExpr, isFloatStore, sb, pad, knownConstants)) { break; } // A translate-time-constant address outside the MMIO window (gather-pipe already handled // above) always takes the flat helper's plain-store arm, so the runtime MMIO test is provably dead. - var provenRamStore = + var provenRamStore = !resumableBody && TryGetKnownEffectiveAddress(store.Address, knownConstants, out var knownStoreAddress) && IsProvenRamStore(knownStoreAddress, store.SizeBytes); string StoreHelper(int sizeBytes, bool isFloat) => provenRamStore @@ -320,6 +328,18 @@ string Raw(string value, bool isFloat) => isFloat { if (TryEmitInlineGuestThunk(addr, sb, pad, types, inlineGuestThunkStackBase)) { + // A restore replaces those FPRs with scalar stack values. + // A resumable body relies on its tags matching the flow, + // which models the same reset. + if (resumableBody && + TryGetInlineGuestThunkSpec(addr, out var thunkSpec) && + thunkSpec.Kind == InlineGuestThunkKind.RestFpr) + { + for (var reg = thunkSpec.StartRegister; reg <= 31; reg++) + { + localPaired[$"f{reg}"] = false; + } + } break; } @@ -338,7 +358,10 @@ string Raw(string value, bool isFloat) => isFloat var stateFreeContract = hasCallSiteVariant ? callSiteVariant!.Contract : stateFreeAbiContracts.GetValueOrDefault(addr); - var usesStateFreeCall = stateFreeSymbol is not null && stateFreeContract is not null && + // State-free signatures and call-site variants are narrowed by + // CFG liveness, which does not see a resume reading an output. + var usesStateFreeCall = !resumableBody && + stateFreeSymbol is not null && stateFreeContract is not null && IsStateFreeCallSafe(stateFreeContract); var resumesThroughLinkRegister = nonReturningCallTargets.Contains(addr) || @@ -409,7 +432,9 @@ _activeResidency is not null && // The callee may resume this body through a local LR // continuation label, so the locals must be refreshed // from the context it just wrote. The exit paths below - // return without writing locals back. + // return without writing locals back. Both paths see the + // same f1 return normalization an ordinary call applies. + EmitNormalizeGuestCallFloatReturn(call.Target, sb, pad, localPaired, guestAbiProvider); AppendReload(sb, pad, boundarySync); var fallbackPad = pad; if (localFallthroughLr.HasValue) @@ -420,11 +445,15 @@ _activeResidency is not null && if (shareLrContinuationDispatch) { + // Shared only when every target expects scalar FPRs. + EmitConvertPairedState( + sb, fallbackPad, localPaired, + new HashSet(StringComparer.OrdinalIgnoreCase)); sb.AppendLine($"{fallbackPad}goto lr_continuation_dispatch;"); } else { - EmitLocalLrContinuationDispatch(sb, fallbackPad, labelNames); + EmitLocalLrContinuationDispatch(sb, fallbackPad, labelNames, localPaired, resumeLabelPairedStates); sb.AppendLine($"{fallbackPad}if (TranslatedFunctionRegistry::FindByAddressPtr(ctx->lr) != nullptr) {{"); sb.AppendLine($"{fallbackPad} InvokeIndirectCpu(ctx->lr, ctx);"); sb.AppendLine($"{fallbackPad}}}"); @@ -693,7 +722,9 @@ private static bool TryEmitKnownGpuFifoStore( private static void EmitLocalLrContinuationDispatch( StringBuilder sb, string pad, - IReadOnlyDictionary labelNames) + IReadOnlyDictionary labelNames, + IReadOnlyDictionary? callSitePaired = null, + IReadOnlyDictionary>? labelPairedStates = null) { var localContinuations = labelNames .Where(static pair => TryParseAddress(pair.Key, out _)) @@ -715,6 +746,11 @@ private static void EmitLocalLrContinuationDispatch( { RecordEmittedLocalLabel(label); sb.AppendLine($"{pad}case 0x{address:X8}u:"); + // Each case is an edge from the call site into that label. + if (callSitePaired is not null && labelPairedStates is not null) + { + EmitConvertPairedState(sb, pad + " ", callSitePaired, labelPairedStates[label]); + } sb.AppendLine($"{pad} goto {label};"); } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Expressions.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Expressions.cs index 6884b6afb..0913ff0f8 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Expressions.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Expressions.cs @@ -339,7 +339,13 @@ private static string RemapAddress( if (effective >= remap.LinkBase && effective < linkEnd) { var relocated = checked(remap.GuestBase + (effective - remap.LinkBase)); - return $"0x{relocated:X8}u"; + if (!remap.GuardAgainstResume) + { + return $"0x{relocated:X8}u"; + } + + var runtime = Address(addr, types); + return $"(static_cast({runtime}) == 0x{effective:X8}u ? 0x{relocated:X8}u : static_cast({runtime}))"; } } diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Locals.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Locals.cs index abf54e5a5..2fa7ef79c 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Locals.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.Locals.cs @@ -187,7 +187,8 @@ private static string ApplyGuardedGqrFunctionVersioning( private sealed record PairedFlowStateMaps( Dictionary> In, - Dictionary> Out); + Dictionary> Out, + Dictionary<(string Block, int Index), HashSet> Before); private static HashSet CollectNonRegisterLocals(IrFunction function) { @@ -276,7 +277,9 @@ private static HashSet MaterializePairedFloatSet(uint mask, StringCompar private static PairedFlowStateMaps ComputePairedFlowStates( IrFunction func, IrCfg cfg, - IGuestFunctionAbiProvider guestAbiProvider) + IGuestFunctionAbiProvider guestAbiProvider, + IReadOnlyDictionary>? recordStatesBefore = null, + bool inlineGuestThunksKeepTags = false) { var comparer = StringComparer.OrdinalIgnoreCase; // f0..f31 occupy bits 0..31. The fixpoint used to seed a fresh @@ -316,6 +319,167 @@ private static PairedFlowStateMaps ComputePairedFlowStates( .ToList(); } + // One instruction's effect on the paired mask. The phi case reads the + // predecessors' current out masks, so it is only exact once they settle. + uint TransferPairedMask(IrInstruction ins, uint newOut) + { + switch (ins) + { + case IrPhi phi: + { + var destBit = PairedFloatRegisterBit(phi.Destination); + if (destBit < 0) + { + break; + } + + var allPaired = true; + foreach (var (predLabel, src) in phi.Sources) + { + var srcBit = PairedFloatRegisterBit(src); + if (srcBit < 0) + { + allPaired = false; + break; + } + + if (!blockIndices.TryGetValue(predLabel, out var predIndex) || + (outMasks[predIndex] & (1u << srcBit)) == 0) + { + allPaired = false; + break; + } + } + + if (allPaired) + { + newOut |= 1u << destBit; + } + else + { + newOut &= ~(1u << destBit); + } + break; + } + case IrCall call: + { + // The emitter expands save/restore thunks in place. Only + // a restore changes tags, to scalar for what it reloads. A + // resume label's recorded state has to match the emitter. + if (inlineGuestThunksKeepTags && + TryParseAddress(call.Target, out var thunkAddress) && + TryGetInlineGuestThunkSpec(thunkAddress, out var thunkSpec)) + { + if (thunkSpec.Kind == InlineGuestThunkKind.RestFpr) + { + for (var reg = thunkSpec.StartRegister; reg <= 31; reg++) + { + newOut &= ~(1u << reg); + } + } + break; + } + + if (IsGuestCallTarget(call.Target)) + { + // Normalize proven scalar arguments and treat f1 as the + // conventional scalar return. Preserve f2-f13 representation + // tags when the callee leaves their payloads untouched. + newOut = ClearPairedGuestScalarFloatArguments( + call.Target, call.Arguments, newOut, guestAbiProvider); + newOut &= ~f1Bit; + } + + if (!string.IsNullOrWhiteSpace(call.Destination)) + { + var destBit = PairedFloatRegisterBit(call.Destination); + if (destBit < 0) + { + break; + } + if (IsPairedProducerTarget(call.Target)) + { + newOut |= 1u << destBit; + } + else + { + newOut &= ~(1u << destBit); + } + } + break; + } + case IrIndirectCall icall when !string.IsNullOrWhiteSpace(icall.Destination): + { + newOut &= ~f1Bit; + var destBit = PairedFloatRegisterBit(icall.Destination); + if (destBit >= 0) + { + newOut &= ~(1u << destBit); + } + break; + } + case IrIndirectCall icall: + newOut &= ~f1Bit; + break; + case IrAssign assign: + { + var destBit = PairedFloatRegisterBit(assign.Destination); + if (destBit < 0) + { + break; + } + + if (assign.Value.Kind == "register" && assign.Value.RegisterName != null) + { + var srcBit = PairedFloatRegisterBit(assign.Value.RegisterName); + if (srcBit >= 0 && (newOut & (1u << srcBit)) != 0) + { + newOut |= 1u << destBit; + } + else + { + newOut &= ~(1u << destBit); + } + } + else + { + newOut &= ~(1u << destBit); + } + break; + } + case IrBinary bin: + { + var destBit = PairedFloatRegisterBit(bin.Destination); + if (destBit >= 0) + { + newOut &= ~(1u << destBit); + } + break; + } + case IrLoad load: + newOut = ClearPairedBit(newOut, load.Destination); + break; + case IrResolvedLoad load: + newOut = ClearPairedBit(newOut, load.Destination); + break; + case IrResolvedLoadPair pair: + newOut = ClearPairedBit(newOut, pair.FirstDestination); + newOut = ClearPairedBit(newOut, pair.SecondDestination); + break; + case IrResolvedPsqLoad load: + { + var destBit = PairedFloatRegisterBit(load.Destination); + if (destBit >= 0) + { + newOut |= 1u << destBit; + } + break; + } + } + + return newOut; + } + var changed = true; while (changed) { @@ -343,142 +507,7 @@ private static PairedFlowStateMaps ComputePairedFlowStates( var newOut = newIn; foreach (var ins in block.Instructions) { - switch (ins) - { - case IrPhi phi: - { - var destBit = PairedFloatRegisterBit(phi.Destination); - if (destBit < 0) - { - break; - } - - var allPaired = true; - foreach (var (predLabel, src) in phi.Sources) - { - var srcBit = PairedFloatRegisterBit(src); - if (srcBit < 0) - { - allPaired = false; - break; - } - - if (!blockIndices.TryGetValue(predLabel, out var predIndex) || - (outMasks[predIndex] & (1u << srcBit)) == 0) - { - allPaired = false; - break; - } - } - - if (allPaired) - { - newOut |= 1u << destBit; - } - else - { - newOut &= ~(1u << destBit); - } - break; - } - case IrCall call: - { - if (IsGuestCallTarget(call.Target)) - { - // Normalize proven scalar arguments and treat f1 as the - // conventional scalar return. Preserve f2-f13 representation - // tags when the callee leaves their payloads untouched. - newOut = ClearPairedGuestScalarFloatArguments( - call.Target, call.Arguments, newOut, guestAbiProvider); - newOut &= ~f1Bit; - } - - if (!string.IsNullOrWhiteSpace(call.Destination)) - { - var destBit = PairedFloatRegisterBit(call.Destination); - if (destBit < 0) - { - break; - } - if (IsPairedProducerTarget(call.Target)) - { - newOut |= 1u << destBit; - } - else - { - newOut &= ~(1u << destBit); - } - } - break; - } - case IrIndirectCall icall when !string.IsNullOrWhiteSpace(icall.Destination): - { - newOut &= ~f1Bit; - var destBit = PairedFloatRegisterBit(icall.Destination); - if (destBit >= 0) - { - newOut &= ~(1u << destBit); - } - break; - } - case IrIndirectCall icall: - newOut &= ~f1Bit; - break; - case IrAssign assign: - { - var destBit = PairedFloatRegisterBit(assign.Destination); - if (destBit < 0) - { - break; - } - - if (assign.Value.Kind == "register" && assign.Value.RegisterName != null) - { - var srcBit = PairedFloatRegisterBit(assign.Value.RegisterName); - if (srcBit >= 0 && (newOut & (1u << srcBit)) != 0) - { - newOut |= 1u << destBit; - } - else - { - newOut &= ~(1u << destBit); - } - } - else - { - newOut &= ~(1u << destBit); - } - break; - } - case IrBinary bin: - { - var destBit = PairedFloatRegisterBit(bin.Destination); - if (destBit >= 0) - { - newOut &= ~(1u << destBit); - } - break; - } - case IrLoad load: - newOut = ClearPairedBit(newOut, load.Destination); - break; - case IrResolvedLoad load: - newOut = ClearPairedBit(newOut, load.Destination); - break; - case IrResolvedLoadPair pair: - newOut = ClearPairedBit(newOut, pair.FirstDestination); - newOut = ClearPairedBit(newOut, pair.SecondDestination); - break; - case IrResolvedPsqLoad load: - { - var destBit = PairedFloatRegisterBit(load.Destination); - if (destBit >= 0) - { - newOut |= 1u << destBit; - } - break; - } - } + newOut = TransferPairedMask(ins, newOut); } if (newIn != inMasks[b]) @@ -503,7 +532,32 @@ private static PairedFlowStateMaps ComputePairedFlowStates( outMap[blocks[i].Label] = MaterializePairedFloatSet(outMasks[i], comparer); } - return new PairedFlowStateMaps(inMap, outMap); + // The settled state in front of selected instructions, for labels a + // resume can reach in the middle of a block. + var beforeMap = new Dictionary<(string Block, int Index), HashSet>(); + if (recordStatesBefore is not null) + { + for (var b = 0; b < blockCount; b++) + { + var block = blocks[b]; + if (!recordStatesBefore.TryGetValue(block.Label, out var indices) || indices.Count == 0) + { + continue; + } + + var mask = inMasks[b]; + for (var i = 0; i < block.Instructions.Count; i++) + { + if (indices.Contains(i)) + { + beforeMap[(block.Label, i)] = MaterializePairedFloatSet(mask, comparer); + } + mask = TransferPairedMask(block.Instructions[i], mask); + } + } + } + + return new PairedFlowStateMaps(inMap, outMap, beforeMap); } private static uint ClearPairedGuestScalarFloatArguments( diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs index e67488403..2e0cee994 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.cs @@ -121,21 +121,29 @@ public CxxEmissionResult EmitWithFacts(uint entryPoint, var leafStackFunction = elideLeafAbiSpills ? ElideLeafStackFprSaveRestore(leafGprStackFunction, types) : leafGprStackFunction; - var func = ElideDeadPureLocalTemps( - ElideFctiwStackLowWordLoads( - ElideOverwrittenPsqStackLoads(leafStackFunction), - types)); + // A body that a callee can resume at any instruction label is entered on + // paths the static CFG does not see. Facts carried from one guest + // instruction to a later one do not hold on those paths, so the passes + // that forward them stay off for it. Removing an overwritten psq stack + // load also rests on a stack proof a resume can skip, and the load's + // memory access is still observable then. + var resumableBody = HasOutOfBandLocalEntry(leafStackFunction, nonReturningCallTargets, lrContinuationCallTargets); + var func = ElideDeadPureLocalTemps(resumableBody + ? leafStackFunction + : ElideFctiwStackLowWordLoads(ElideOverwrittenPsqStackLoads(leafStackFunction), types)); var isGuestCallFree = FunctionEligibleForLeafRegisterCache(func); var hasProvenKernelEntry = isGuestCallFree && gqrEntryConstants is { Count: > 0 }; - func = GqrConstantPropagation.Specialize( - func, gqrEntryConstants, gqrCalleeWriteMasks, - gqrConstantsRequireRuntimeGuard); + if (!resumableBody) + { + func = GqrConstantPropagation.Specialize( + func, gqrEntryConstants, gqrCalleeWriteMasks, + gqrConstantsRequireRuntimeGuard); + } // Call-free proven kernels amortize very small ranges. Ordinary functions // use the epoch-safe scalar form and a higher threshold to avoid adding // pointer locals for incidental field pairs. Range resolves and access pairs - // are placed by the static CFG, so a body that can be resumed at any - // instruction label keeps its plain accesses. - if (!HasOutOfBandLocalEntry(func, nonReturningCallTargets, lrContinuationCallTargets)) + // are placed by the static CFG, so a resumable body keeps its plain accesses. + if (!resumableBody) { func = GuestMemoryRangeLowering.Lower( func, minimumAccesses: hasProvenKernelEntry ? 2 : 8, ignoreDisabledPcTraces: true, @@ -143,6 +151,33 @@ public CxxEmissionResult EmitWithFacts(uint entryPoint, } var architecturalFunc = func; var guestAbiContract = GuestAbiContractAnalyzer.Analyze(architecturalFunc, guestAbiContracts); + if (resumableBody) + { + // The analysis follows CFG paths only. A resume can read a register + // past the write that path saw, so callers get a full fence instead. + guestAbiContract = guestAbiContract with + { + GprReadBeforeWriteMask = uint.MaxValue, + FprReadBeforeWriteMask = uint.MaxValue, + CrReadBeforeWriteMask = byte.MaxValue, + ReadsXerBeforeWrite = true, + ReadsCtrBeforeWrite = true, + ReadsLrBeforeWrite = true, + ReadsFpscrBeforeWrite = true, + GqrReadBeforeWriteMask = byte.MaxValue, + HidReadBeforeWriteMask = byte.MaxValue, + GprDefiniteWriteMask = 0, + FprDefiniteWriteMask = 0, + CrDefiniteWriteMask = 0, + DefinitelyWritesXer = false, + DefinitelyWritesCtr = false, + DefinitelyWritesLr = false, + DefinitelyWritesFpscr = false, + GqrDefiniteWriteMask = 0, + HidDefiniteWriteMask = 0, + BoundaryFlags = guestAbiContract.BoundaryFlags | GuestCallBoundaryFlags.RequiresCompleteContext + }; + } var cfg = IrCfg.Build(func); // Condition-flag elision is planned once against the final IR so every // emission of this body (recording pass, public entry, state-free @@ -154,19 +189,10 @@ public CxxEmissionResult EmitWithFacts(uint entryPoint, // tuple for every instruction of every emission of this body. var suppressedInstructionMasks = BuildSuppressedInstructionMasks(func, flagElision); var cxxName = SanitizeIdentifier(signature.Name); - var pairedFlow = ComputePairedFlowStates(func, cfg, _guestAbiProvider); - var pairedIn = pairedFlow.In; - var pairedOut = pairedFlow.Out; - var pairedEdgeLiveness = ComputeRegisterLiveness( - func, cfg, - new LeafRegisterCache( - new SortedSet(), new SortedSet(Enumerable.Range(0, 32)), - new SortedSet(), new SortedSet()), - _guestAbiProvider); var leafPreservedFprMask = LeafStackPreservedFprMask(architecturalFunc); var nonvolatileFprWriteMask = FunctionNonvolatileFprWriteMask(architecturalFunc) & ~leafPreservedFprMask; var preservesNonvolatileFprs = nonvolatileFprWriteMask == 0; - var stackFacts = StackAddressFacts.Build(func); + var stackFacts = StackAddressFacts.Build(func, trustRegisterCopies: !resumableBody); // Register residency supersedes the leaf register cache (localizes state via the // expression layer instead of rewriting text, and covers non-leaf bodies too); // residency owns every primary body, so the text-rewriting cache survives only for the state-free clone below. @@ -258,7 +284,6 @@ block.Instructions[index] is not IrCall call || .Any(call => TryParseAddress(call.Target, out var target) && (nonReturningCallTargets.Contains(target) || lrContinuationCallTargets.Contains(target))); - var shareLrContinuationDispatch = continuationCallCount > 1; if (needsInstructionContinuationLabels) { foreach (var trace in func.Blocks.SelectMany(static block => block.Instructions).OfType()) @@ -272,6 +297,53 @@ block.Instructions[index] is not IrCall call || } } } + var continuationLabelStarts = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var block in func.Blocks) + { + if (MapContinuationLabelsToInstructionStarts(block.Instructions, instructionContinuationLabels) is { } starts) + { + continuationLabelStarts[block.Label] = starts; + } + } + var pairedFlow = ComputePairedFlowStates( + func, cfg, _guestAbiProvider, + continuationLabelStarts.ToDictionary( + static pair => pair.Key, + static pair => (IReadOnlySet)pair.Value.Keys.ToHashSet(), + StringComparer.OrdinalIgnoreCase), + inlineGuestThunksKeepTags: resumableBody); + var pairedIn = pairedFlow.In; + var pairedOut = pairedFlow.Out; + var pairedEdgeLiveness = ComputeRegisterLiveness( + func, cfg, + new LeafRegisterCache( + new SortedSet(), new SortedSet(Enumerable.Range(0, 32)), + new SortedSet(), new SortedSet()), + _guestAbiProvider); + // A resume can jump to any label from any continuation call site. Each + // label keeps the paired state its CFG predecessors give it, and the + // dispatch converts the call site's state to that one, the same way a + // CFG edge into the label would. + Dictionary>? resumeLabelPairedStates = null; + if (needsInstructionContinuationLabels) + { + resumeLabelPairedStates = new Dictionary>(StringComparer.Ordinal); + foreach (var block in func.Blocks) + { + resumeLabelPairedStates[labelNames[block.Label]] = pairedIn[block.Label]; + } + foreach (var (blockLabel, starts) in continuationLabelStarts) + { + foreach (var (index, label) in starts) + { + resumeLabelPairedStates[label] = pairedFlow.Before[(blockLabel, index)]; + } + } + } + // The shared dispatcher cannot know which call site it came from, so it + // is only used when every target label expects scalar FPRs. + var shareLrContinuationDispatch = continuationCallCount > 1 && + (resumeLabelPairedStates?.Values.All(static state => state.Count == 0) ?? true); var nextBlockLabels = BuildNextBlockLabelMap(func); var knownConstants = BuildKnownUIntConstants(func); // Burst coalescing is planned once against the final IR so every @@ -282,11 +354,12 @@ block.Instructions[index] is not IrCall call || // The state-free clone is derived by rewriting this body's text, and it // rewrites the ctx-carrying PSQ helper spellings into the explicit-state // ones. Hoisting changes those spellings, so leave it off for the bodies - // that clone is made from. + // that clone is made from. A resume label can sit past a reload, so a + // resumable body reads its GQRs at each use. var hoistedGqrIndices = CollectHoistableGqrIndices( - func, stackFacts, !emitStateFreeLeafVariant); + func, stackFacts, !emitStateFreeLeafVariant && !resumableBody); var linkedAddressRemap = moduleLinkBase.HasValue && moduleGuestBase.HasValue && moduleLinkedCodeSize != 0 - ? new LinkedAddressRemap(moduleLinkBase.Value, moduleGuestBase.Value, moduleLinkedCodeSize) + ? new LinkedAddressRemap(moduleLinkBase.Value, moduleGuestBase.Value, moduleLinkedCodeSize, resumableBody) : null; // The complete function body is produced by this local function so it // can be emitted more than once with a different residency mode. The @@ -384,14 +457,23 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool var blockSuppressed = suppressedInstructionMasks.TryGetValue(block.Label, out var suppressedMask) ? suppressedMask : null; - var continuationLabelsByStart = MapContinuationLabelsToInstructionStarts( - block.Instructions, instructionContinuationLabels); + continuationLabelStarts.TryGetValue(block.Label, out var continuationLabelsByStart); for (var i = 0; i < nonTerminatorCount; i++) { if (continuationLabelsByStart is not null && continuationLabelsByStart.TryGetValue(i, out var continuationLabel)) { + // A resume reaches this label from another call site + // in the state the flow gives it, so the fallthrough is + // brought to that same state. + var labelPaired = resumeLabelPairedStates![continuationLabel]; + EmitConvertPairedState(body, " ", localPaired, labelPaired); + localPaired.Clear(); + foreach (var reg in labelPaired) + { + localPaired[reg] = true; + } RecordEmittedLocalLabel(continuationLabel); body.AppendLine($"{continuationLabel}: ;"); } @@ -422,7 +504,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool _activeGpuFifoBurstSlot = gpuFifoBurstPlan.Slot(block.Label, i); try { - EmitInstruction(block.Label, directCallOrdinal, block.Instructions[i], body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase, localFallthroughLr, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch); + EmitInstruction(block.Label, directCallOrdinal, block.Instructions[i], body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase, localFallthroughLr, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch, resumeLabelPairedStates); } finally { @@ -440,7 +522,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool switch (term) { case IrUndefined undef: - EmitInstruction(block.Label, -1, undef, body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase: false, localFallthroughLr: null, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch); + EmitInstruction(block.Label, -1, undef, body, bufferBaseLength, 1, cfg, labelNames, types, signature, localPaired, _guestAbiProvider, knownConstants, localConstants, linkedAddressRemap, nonReturningCallTargets, lrContinuationCallTargets, stackFacts, inlineGuestThunkStackBase: false, localFallthroughLr: null, guestAbiContracts, stateFreeAbiContracts, stateFreeCallSymbols, stateFreeCallSiteVariants, modOverridableCallTargets, shareLrContinuationDispatch, resumeLabelPairedStates); AppendFlush(body, " "); body.AppendLine(" return;"); break; @@ -451,7 +533,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool break; case IrJump jump: - EmitNormalizePairedStateOnEdge(body, " ", block.Label, jump.TargetLabel, pairedOut, pairedIn, pairedEdgeLiveness.In); + EmitNormalizePairedStateOnEdge(body, " ", block.Label, jump.TargetLabel, pairedOut, pairedIn, resumableBody ? null : pairedEdgeLiveness.In); EmitGotoUnlessFallthrough(body, " ", block.Label, jump.TargetLabel, nextBlockLabels, labelNames, signature.Name); break; @@ -466,10 +548,10 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool // empty "} else {}". Build it first and keep the else // only when it actually carries something. var falseEdge = new StringBuilder(); - EmitNormalizePairedStateOnEdge(falseEdge, " ", block.Label, br.FalseLabel, pairedOut, pairedIn, pairedEdgeLiveness.In); + EmitNormalizePairedStateOnEdge(falseEdge, " ", block.Label, br.FalseLabel, pairedOut, pairedIn, resumableBody ? null : pairedEdgeLiveness.In); EmitGotoUnlessFallthrough(falseEdge, " ", block.Label, br.FalseLabel, nextBlockLabels, labelNames, signature.Name); body.AppendLine($" if ({cond}) {{"); - EmitNormalizePairedStateOnEdge(body, " ", block.Label, br.TrueLabel, pairedOut, pairedIn, pairedEdgeLiveness.In); + EmitNormalizePairedStateOnEdge(body, " ", block.Label, br.TrueLabel, pairedOut, pairedIn, resumableBody ? null : pairedEdgeLiveness.In); EmitGotoUnlessFallthrough(body, " ", block.Label, br.TrueLabel, nextBlockLabels, labelNames, signature.Name); if (falseEdge.Length != 0) { @@ -486,7 +568,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool { var caseLabel = RequireLabel(jt.TargetLabel, labelNames, block.Label, signature.Name); body.AppendLine($" case 0x{jt.TargetAddress:X8}u:"); - EmitNormalizePairedStateOnEdge(body, " ", block.Label, jt.TargetLabel, pairedOut, pairedIn, pairedEdgeLiveness.In); + EmitNormalizePairedStateOnEdge(body, " ", block.Label, jt.TargetLabel, pairedOut, pairedIn, resumableBody ? null : pairedEdgeLiveness.In); body.AppendLine($" goto {caseLabel};"); body.AppendLine(" break;"); } @@ -505,7 +587,7 @@ string EmitFunctionBody(RegisterResidency? residency, int bufferBaseLength, bool var succ = cfg.Successors(block.Label).FirstOrDefault(); if (!string.IsNullOrWhiteSpace(succ)) { - EmitNormalizePairedStateOnEdge(body, " ", block.Label, succ!, pairedOut, pairedIn, pairedEdgeLiveness.In); + EmitNormalizePairedStateOnEdge(body, " ", block.Label, succ!, pairedOut, pairedIn, resumableBody ? null : pairedEdgeLiveness.In); EmitGotoUnlessFallthrough(body, " ", block.Label, succ!, nextBlockLabels, labelNames, signature.Name); } else @@ -834,7 +916,10 @@ private static void UpdateKnownUIntConstants(IrInstruction instruction, Dictiona } } - private sealed record LinkedAddressRemap(uint LinkBase, uint GuestBase, uint CodeSize); + // GuardAgainstResume: the constant behind a remap may come from an + // instruction a resume skipped, so the literal is only used when the runtime + // address still equals that constant. + private sealed record LinkedAddressRemap(uint LinkBase, uint GuestBase, uint CodeSize, bool GuardAgainstResume); private static bool TryEvaluateUIntBinary( IrBinary binary, @@ -1043,6 +1128,39 @@ private static uint NonvolatileFprMask(string registerName) return index is >= 14 and <= 31 ? 1u << index : 0; } + /// + /// Converts every FPR whose tag differs between the current state and the + /// state a resume label expects. Liveness is not consulted, because CFG + /// liveness does not see the resume paths. + /// + private static void EmitConvertPairedState( + StringBuilder sb, + string pad, + IReadOnlyDictionary fromPaired, + IReadOnlySet toPaired) + { + var candidateRegs = new HashSet(toPaired, StringComparer.OrdinalIgnoreCase); + candidateRegs.UnionWith(fromPaired.Where(static pair => pair.Value).Select(static pair => pair.Key)); + foreach (var reg in candidateRegs + .Where(IsFloatRegister) + .OrderBy(ParseFloatRegisterIndex) + .ThenBy(static reg => reg, StringComparer.OrdinalIgnoreCase)) + { + var fromIsPaired = fromPaired.TryGetValue(reg, out var paired) && paired; + var toIsPaired = toPaired.Contains(reg); + if (fromIsPaired == toIsPaired) + continue; + + var regIndex = ParseFloatRegisterIndex(reg); + var regExpr = _activeResidency is null + ? $"ctx->fpr[{regIndex}].d" + : _activeResidency.FprScalar(regIndex, written: true); + sb.AppendLine(toIsPaired + ? $"{pad}{regExpr} = PPC_PsFromScalarInline({regExpr});" + : $"{pad}{regExpr} = PPC_PsToScalarInline({regExpr});"); + } + } + private static void EmitNormalizePairedStateOnEdge( StringBuilder sb, string pad, @@ -1050,7 +1168,7 @@ private static void EmitNormalizePairedStateOnEdge( string toLabel, IReadOnlyDictionary> pairedOut, IReadOnlyDictionary> pairedIn, - IReadOnlyDictionary> liveIn) + IReadOnlyDictionary>? liveIn) { pairedOut.TryGetValue(fromLabel, out var fromSet); pairedIn.TryGetValue(toLabel, out var toSet); @@ -1060,10 +1178,15 @@ private static void EmitNormalizePairedStateOnEdge( var candidateRegs = new HashSet(fromSet, StringComparer.OrdinalIgnoreCase); candidateRegs.UnionWith(toSet); - if (liveIn.TryGetValue(toLabel, out var targetLive)) - candidateRegs.IntersectWith(targetLive); - else - candidateRegs.Clear(); + // Without liveness every mismatch is converted: CFG liveness does not + // see a resume that reads the register later. + if (liveIn is not null) + { + if (liveIn.TryGetValue(toLabel, out var targetLive)) + candidateRegs.IntersectWith(targetLive); + else + candidateRegs.Clear(); + } foreach (var reg in candidateRegs .Where(IsFloatRegister) diff --git a/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs b/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs index 0369e7d52..0c0d95e4a 100644 --- a/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs +++ b/translator/tests/Translator.Tests/GpuFifoBurstCodeGenTests.cs @@ -253,9 +253,10 @@ public void UnsupportedFloatWidthFallsBackToGuestMemory() [Theory] [InlineData(false)] [InlineData(true)] - public void ResumableBodiesKeepIndividualFifoStores(bool resumable) + public void ResumableBodiesKeepCheckedFifoStores(bool resumable) { - // A hook resuming at a later store would otherwise flush burst slots it never wrote. + // A hook resuming at a later store would otherwise flush burst slots it never wrote, + // and a resumed body cannot prove the constant address still holds. var body = new List { new IrAssign("lr", IrValue.Imm(0x80001008)), @@ -286,8 +287,8 @@ public void ResumableBodiesKeepIndividualFifoStores(bool resumable) return; } - Assert.DoesNotContain("GX_HLE_FIFO_WriteBurst(", code, StringComparison.Ordinal); - Assert.Equal(4, CountOccurrences(code, "GX_HLE_FIFO_Write32(")); + Assert.DoesNotContain("GX_HLE_FIFO_Write", code, StringComparison.Ordinal); + Assert.Equal(4, CountOccurrences(code, "MemoryInline::FlatWrite32(")); } private static int CountOccurrences(string text, string value) diff --git a/translator/tests/Translator.Tests/ResumableBodyCodeGenTests.cs b/translator/tests/Translator.Tests/ResumableBodyCodeGenTests.cs new file mode 100644 index 000000000..46402b45c --- /dev/null +++ b/translator/tests/Translator.Tests/ResumableBodyCodeGenTests.cs @@ -0,0 +1,472 @@ +using Translator.Core.Analysis; +using Translator.Core.Analysis.Ssa; +using Translator.Core.Analysis.Representation; +using Translator.Core.CodeGen; +using Translator.Core.Ir; +using Translator.Core.Representation; +using Xunit; + +namespace Translator.Tests; + +// A callee can resume these bodies at any instruction label, so nothing a pass +// learned from the instructions above a label may be trusted below it. +public class ResumableBodyCodeGenTests +{ + private const uint Entry = 0x80001000u; + private const uint Hook = 0x8179AC3Cu; + + // The hook call comes first, so every later guest instruction is a resume target. + private static IrFunction Body(string name, params IrInstruction[][] guestInstructions) + { + var instructions = new List + { + new IrAssign("lr", IrValue.Imm(unchecked((int)(Entry + 4)))), + new IrCall(string.Empty, $"0x{Hook:X8}", Array.Empty()), + new IrTracePpc(Entry, "bl", "0x48000001") + }; + for (var i = 0; i < guestInstructions.Length; i++) + { + instructions.AddRange(guestInstructions[i]); + instructions.Add(new IrTracePpc(Entry + 4 + (uint)i * 4, "op", "0x60000000")); + } + instructions.Add(new IrReturn(null)); + return new IrFunction(name, $"0x{Entry:X8}", + new[] { new IrBasicBlock($"0x{Entry:X8}", instructions) }); + } + + private static string Emit( + IrFunction function, + bool resumable, + Dictionary? registers = null, + uint? moduleLinkBase = null, + uint? moduleGuestBase = null, + uint moduleLinkedCodeSize = 0, + IReadOnlyDictionary? guestAbiContracts = null, + IReadOnlyDictionary? stateFreeAbiContracts = null, + IReadOnlyDictionary? stateFreeCallSymbols = null) => + new CxxLinearCodeGenerator().Emit( + Entry, + new SsaTransformer().Convert(function), + new FunctionAbiClassification(function.Name, ValueRepresentation.Void), + new RepresentationEnvironment(registers ?? new Dictionary()), + lrContinuationCallTargets: resumable ? new HashSet { Hook } : new HashSet(), + moduleLinkBase: moduleLinkBase, + moduleGuestBase: moduleGuestBase, + moduleLinkedCodeSize: moduleLinkedCodeSize, + guestAbiContracts: guestAbiContracts, + stateFreeAbiContracts: stateFreeAbiContracts, + stateFreeCallSymbols: stateFreeCallSymbols); + + // An empty callee: reads and writes nothing. + private static GuestAbiContract EmptyContract() => GuestAbiContractAnalyzer.Analyze( + new IrFunction("empty", "entry", new[] { new IrBasicBlock("entry", new IrInstruction[] { new IrReturn(null) }) })); + + private static string ResumedAt(string code, uint address) + { + var label = code.IndexOf($"loc_{address:X8}: ;", StringComparison.Ordinal); + Assert.True(label >= 0, code); + return code[label..]; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void FctiwLowWordIsReloadedFromTheStackSlot(bool resumable) + { + // A resume at the load must read the slot, not a word taken from f0 before the hook ran. + var code = Emit(Body("fctiw_resumable", + new IrInstruction[] { new IrBinary("f0", IrValue.Register("f1"), IrValue.Imm(0), "fctiwz") }, + new IrInstruction[] { new IrStore(new IrAddress("r1", -16), IrValue.Register("f0"), 8) }, + new IrInstruction[] { new IrLoad("r3", new IrAddress("r1", -12), 4) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r1"] = ValueRepresentation.UInt32, + ["r3"] = ValueRepresentation.UInt32, + ["f0"] = ValueRepresentation.Float64, + ["f1"] = ValueRepresentation.Float64 + }); + + if (!resumable) + { + Assert.Contains("fctiwzword", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("fctiwzword", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GqrIsReadAtTheUse(bool resumable) + { + // A resume past the GQR write must not use the value that write would have set. + var code = Emit(Body("gqr_specialize_resumable", + new IrInstruction[] { new IrAssign("r3", IrValue.Imm(0x00040004)) }, + new IrInstruction[] { new IrAssign("gqr2", IrValue.Register("r3")) }, + new IrInstruction[] + { + new IrCall("f1", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(2) }) + }), + resumable); + + if (!resumable) + { + Assert.Contains("Known", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("Known", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GqrIsNotHoistedIntoALocal(bool resumable) + { + var psqLoad = new IrCall("f1", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(0) }); + var code = Emit(Body("gqr_hoist_resumable", + new IrInstruction[] { psqLoad }, + new IrInstruction[] { psqLoad }), + resumable); + + if (!resumable) + { + Assert.Contains("mkw_gqr0", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("mkw_gqr0", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ConstantRamStoreKeepsTheRuntimeMmioCheck(bool resumable) + { + var code = Emit(Body("ram_store_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Imm(unchecked((int)0x80400000u))) }, + new IrInstruction[] { new IrStore(new IrAddress("r4", 0), IrValue.Register("r5"), 4) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["r5"] = ValueRepresentation.UInt32 + }); + + if (!resumable) + { + Assert.Contains("MemoryInline::FlatWriteRam32(", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("MemoryInline::FlatWriteRam32(", code, StringComparison.Ordinal); + Assert.Contains("MemoryInline::FlatWrite32(", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void StackStoreThroughACopyOfR1KeepsTheRuntimeMmioCheck(bool resumable) + { + // A resume at the store skips the copy, so r4 is not known to point at the stack. + var code = Emit(Body("stack_copy_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Register("r1")) }, + new IrInstruction[] { new IrStore(new IrAddress("r4", 0), IrValue.Register("r5"), 4) }, + new IrInstruction[] { new IrStore(new IrAddress("r1", 8), IrValue.Register("r5"), 4) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r1"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["r5"] = ValueRepresentation.UInt32 + }); + + if (!resumable) + { + Assert.Equal(2, CountOccurrences(code, "MemoryInline::FlatWriteRam32(")); + return; + } + + // r1 itself is still the stack pointer after a resume. + Assert.Equal(1, CountOccurrences(code, "MemoryInline::FlatWriteRam32(")); + Assert.Contains("MemoryInline::FlatWrite32(r4,", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PsqAccessThroughACopyOfR1UsesTheCheckedHelpers(bool resumable) + { + // The stack helpers skip the executable-write and deferred-read handling, + // which a resume that skipped the copy still needs. + var code = Emit(Body("psq_stack_copy_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Register("r1")) }, + new IrInstruction[] + { + new IrCall("f1", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(0) }) + }, + new IrInstruction[] + { + new IrCall(string.Empty, "PPC_PsqSt", + new[] { IrValue.Register("r4"), IrValue.Register("f1"), IrValue.Imm(0), IrValue.Imm(0) }) + }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r1"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["f1"] = ValueRepresentation.Float64 + }); + + if (!resumable) + { + Assert.Contains("PPC_PsqLStackInline<", code, StringComparison.Ordinal); + Assert.Contains("PPC_PsqStStackInline<", code, StringComparison.Ordinal); + return; + } + + Assert.DoesNotContain("StackInline<", code, StringComparison.Ordinal); + Assert.Contains("PPC_PsqLInline<", code, StringComparison.Ordinal); + Assert.Contains("PPC_PsqStInline<", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void OverwrittenPsqStackLoadThroughACopyOfR1IsKept(bool resumable) + { + // A resume at the psq_l skips the copy, so its read is a real guest access. + var code = Emit(Body("psq_overwritten_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Register("r1")) }, + new IrInstruction[] + { + new IrBinary("psq_addr", IrValue.Register("r4"), IrValue.Imm(8), "add"), + new IrCall("f14", "PPC_PsqL", new[] { IrValue.Register("psq_addr"), IrValue.Imm(0), IrValue.Imm(0) }) + }, + new IrInstruction[] { new IrLoad("f14", new IrAddress("r1", 0), 8) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r1"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["psq_addr"] = ValueRepresentation.UInt32, + ["f14"] = ValueRepresentation.Float64 + }); + + if (!resumable) + { + Assert.DoesNotContain("PPC_PsqL", code, StringComparison.Ordinal); + return; + } + + Assert.Contains("PPC_PsqLInline<", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PublishedContractIsAFullFence(bool resumable) + { + // Resuming at the copy skips the write, so incoming r4 is read. + var code = Emit(Body("contract_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Imm(1)) }, + new IrInstruction[] { new IrAssign("r3", IrValue.Register("r4")) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r3"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32 + }, + guestAbiContracts: new Dictionary { [Hook] = EmptyContract() }); + + if (!resumable) + { + Assert.Contains("fence=0", code, StringComparison.Ordinal); + Assert.DoesNotContain("gpr_read=0xFFFFFFFF", code, StringComparison.Ordinal); + return; + } + + Assert.Contains("gpr_read=0xFFFFFFFF", code, StringComparison.Ordinal); + Assert.Contains("fence=1", code, StringComparison.Ordinal); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CallsKeepTheFullContextBoundary(bool resumable) + { + // A state-free signature only returns what CFG liveness wants, and a resume can read more. + const uint callee = 0x80002000u; + const string symbol = "callee_state_free"; + var code = Emit(Body("state_free_resumable", + new IrInstruction[] { new IrCall(string.Empty, $"0x{callee:X8}", Array.Empty()) }), + resumable, + new Dictionary { ["lr"] = ValueRepresentation.UInt32 }, + guestAbiContracts: new Dictionary + { + [Hook] = EmptyContract(), + [callee] = EmptyContract() + }, + stateFreeAbiContracts: new Dictionary { [callee] = EmptyContract() }, + stateFreeCallSymbols: new Dictionary { [callee] = symbol }); + + // The extern declaration is emitted either way. + if (!resumable) + { + Assert.True(CountOccurrences(code, symbol + "(") > 1, code); + return; + } + + Assert.Equal(1, CountOccurrences(code, symbol + "(")); + Assert.Contains($"InvokeDirectCpu<0x{callee:X8}u>(ctx);", code, StringComparison.Ordinal); + } + + [Fact] + public void InlineFprRestoreThunkRetagsTheRestoredRegisters() + { + // _rest_fpr_23 reloads f23 as a scalar, so the paired tag from the psq_l is gone. + var code = Emit(Body("thunk_restore_resumable", + new IrInstruction[] + { + new IrCall("f23", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(0) }) + }, + new IrInstruction[] { new IrCall(string.Empty, "0x80021544", Array.Empty()) }, + new IrInstruction[] { new IrStore(new IrAddress("r5", 0), IrValue.Register("f23"), 4) }), + resumable: true, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["r5"] = ValueRepresentation.UInt32, + ["r11"] = ValueRepresentation.UInt32, + ["f23"] = ValueRepresentation.Float64 + }); + + Assert.DoesNotContain("InvokeDirectCpu<0x80021544u>", code, StringComparison.Ordinal); + // The store label expects scalar f23, so neither the fallthrough nor a resume converts it. + Assert.DoesNotContain("PPC_PsToScalarInline(f23", code, StringComparison.Ordinal); + var caseStart = code.IndexOf($"case 0x{Entry + 12:X8}u:", StringComparison.Ordinal); + Assert.True(caseStart >= 0, code); + var caseBody = code[caseStart..code.IndexOf("goto", caseStart, StringComparison.Ordinal)]; + Assert.DoesNotContain("f23", caseBody, StringComparison.Ordinal); + } + + [Fact] + public void InlineSaveThunkKeepsALivePairedValueIntact() + { + // The thunk is expanded in place and touches no FPR, so nothing may + // convert f1 between the psq_l and the ps_add on the fallthrough. + var code = Emit(Body("thunk_paired_resumable", + new IrInstruction[] + { + new IrCall("f1", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(0) }) + }, + new IrInstruction[] { new IrCall(string.Empty, "0x800215A0", Array.Empty()) }, + new IrInstruction[] + { + new IrCall("f2", "PPC_PsAdd", new[] { IrValue.Register("f1"), IrValue.Register("f1") }) + }), + resumable: true, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["r11"] = ValueRepresentation.UInt32, + ["f1"] = ValueRepresentation.Float64, + ["f2"] = ValueRepresentation.Float64 + }); + + Assert.DoesNotContain("InvokeDirectCpu<0x800215A0u>", code, StringComparison.Ordinal); + Assert.DoesNotContain("PPC_PsToScalarInline(f1", code, StringComparison.Ordinal); + } + + private static int CountOccurrences(string text, string value) + { + var count = 0; + for (var index = text.IndexOf(value, StringComparison.Ordinal); + index >= 0; + index = text.IndexOf(value, index + value.Length, StringComparison.Ordinal)) + { + count++; + } + return count; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void LinkedAddressRemapChecksTheRuntimeAddress(bool resumable) + { + const uint linkBase = 0x803992E0u; + const uint guestBase = 0x81800000u; + var code = Emit(Body("remap_resumable", + new IrInstruction[] { new IrAssign("r4", IrValue.Imm(unchecked((int)(linkBase + 0x10)))) }, + new IrInstruction[] { new IrLoad("r5", new IrAddress("r4", 8), 4) }), + resumable, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["r5"] = ValueRepresentation.UInt32 + }, + moduleLinkBase: linkBase, + moduleGuestBase: guestBase, + moduleLinkedCodeSize: 0x1000); + + if (!resumable) + { + Assert.Contains($"FlatRead32(0x{guestBase + 0x18:X8}u)", code, StringComparison.Ordinal); + return; + } + + // The fallthrough still takes the remapped literal; a resume that skipped + // the constant keeps its own address. + Assert.Contains( + $"== 0x{linkBase + 0x18:X8}u ? 0x{guestBase + 0x18:X8}u : ", + ResumedAt(code, Entry + 8), + StringComparison.Ordinal); + } + + [Fact] + public void ResumeDispatchConvertsToTheTargetLabelsPairedState() + { + // f1 is paired from the psq_l on, but scalar at the hook call. A resume + // to the ps_add label has to enter in the paired form that label expects. + var code = Emit(Body("paired_resumable", + new IrInstruction[] + { + new IrCall("f1", "PPC_PsqL", new[] { IrValue.Register("r4"), IrValue.Imm(0), IrValue.Imm(0) }) + }, + new IrInstruction[] + { + new IrCall("f2", "PPC_PsAdd", new[] { IrValue.Register("f1"), IrValue.Register("f1") }) + }), + resumable: true, + new Dictionary + { + ["lr"] = ValueRepresentation.UInt32, + ["r4"] = ValueRepresentation.UInt32, + ["f1"] = ValueRepresentation.Float64, + ["f2"] = ValueRepresentation.Float64 + }); + + var caseStart = code.IndexOf($"case 0x{Entry + 8:X8}u:", StringComparison.Ordinal); + Assert.True(caseStart >= 0, code); + var caseBody = code[caseStart..code.IndexOf("goto", caseStart, StringComparison.Ordinal)]; + Assert.Contains("PPC_PsFromScalarInline(", caseBody, StringComparison.Ordinal); + + // The fallthrough already holds f1 paired, so it reaches the label unchanged. + var label = code.IndexOf($"loc_{Entry + 8:X8}: ;", StringComparison.Ordinal); + var previousLine = code.LastIndexOf('\n', code.LastIndexOf('\n', label) - 1); + Assert.DoesNotContain("Scalar", code[previousLine..label], StringComparison.Ordinal); + Assert.DoesNotContain("PPC_PsFromScalarInline(", code[label..], StringComparison.Ordinal); + } +} From d347ec135dc3bb5497da7bd152ef9f8bd0805ec4 Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:01:39 +0100 Subject: [PATCH 4/4] full boundary for lr continuation calls --- .../CxxLinearCodeGenerator.EmitInstruction.cs | 4 +- .../RegisterResidencyCodeGenTests.cs | 37 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs index 86285c63b..c5d3cd95f 100644 --- a/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs +++ b/translator/src/Translator.Core/CodeGen/CxxLinearCodeGenerator.EmitInstruction.cs @@ -387,7 +387,9 @@ _activeResidency is not null && EmitNormalizeGuestCallFloatReturn(call.Target, sb, pad, localPaired, guestAbiProvider); break; } - var boundarySync = usesStateFreeCall || argumentNormalization.Length != 0 + // A resume through LR can run code outside this callee's + // contract, including another function reading ctx. + var boundarySync = usesStateFreeCall || argumentNormalization.Length != 0 || resumesThroughLinkRegister ? ResidencyBoundarySync.Full : ResolveDirectCallBoundarySync(addr, guestAbiContracts, modOverridableCallTargets); if (!boundarySync.IsFull && diff --git a/translator/tests/Translator.Tests/RegisterResidencyCodeGenTests.cs b/translator/tests/Translator.Tests/RegisterResidencyCodeGenTests.cs index 67f9a78b8..bb83d8cdf 100644 --- a/translator/tests/Translator.Tests/RegisterResidencyCodeGenTests.cs +++ b/translator/tests/Translator.Tests/RegisterResidencyCodeGenTests.cs @@ -18,7 +18,8 @@ private static string Emit( RepresentationEnvironment? types = null, bool stateFree = false, uint entryPoint = 0x80001000u, - IReadOnlyDictionary? guestAbiContracts = null) + IReadOnlyDictionary? guestAbiContracts = null, + IReadOnlySet? lrContinuationCallTargets = null) { var contract = GuestAbiContractAnalyzer.Analyze(function); return new CxxLinearCodeGenerator().Emit( @@ -27,6 +28,7 @@ private static string Emit( new FunctionAbiClassification(function.Name, ValueRepresentation.Void), types ?? new RepresentationEnvironment(new Dictionary()), guestAbiContracts: guestAbiContracts, + lrContinuationCallTargets: lrContinuationCallTargets, emitStateFreeLeafVariant: stateFree, stateFreeAbiContracts: stateFree ? new Dictionary { [entryPoint] = contract } @@ -308,6 +310,39 @@ public void BoundaryNarrowingSyncsOnlyTheContractIntersectionAtADirectCall() Assert.DoesNotContain("r5 = ctx->gpr[5];", afterCall, StringComparison.Ordinal); } + [Fact] + public void BoundaryNarrowingDoesNotApplyToLrContinuationCalls() + { + // The callee can resume through ctx->lr into code outside its own + // contract, which reads every register from the context. + var function = new IrFunction("lr_continuation_call", "entry", new[] + { + new IrBasicBlock("entry", new IrInstruction[] + { + new IrBinary("r5", IrValue.Register("r5"), IrValue.Register("r6"), "add"), + new IrAssign("lr", IrValue.Imm(0x80001010)), + new IrCall(string.Empty, "0x80002000", Array.Empty()), + new IrBinary("r3", IrValue.Register("r3"), IrValue.Register("r5"), "add"), + new IrReturn(null) + }) + }); + var contracts = new Dictionary + { + [0x80002000u] = NarrowContract(gprWrite: 1u << 3) + }; + + var body = FunctionBody( + Emit( + function, + guestAbiContracts: contracts, + lrContinuationCallTargets: new HashSet { 0x80002000u }), + "lr_continuation_call"); + var call = body.IndexOf("InvokeDirectCpu<0x80002000u>(ctx);", StringComparison.Ordinal); + Assert.True(call > 0); + + Assert.Contains("ctx->gpr[5] = r5;", body[..call], StringComparison.Ordinal); + } + [Fact] public void BoundaryNarrowingFlushesRegistersTheCalleeOnlyMayWrite() {