diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65f570109c..82f13f317d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,6 +377,22 @@ jobs: if: needs.changes.outputs.heavy == 'true' run: bash scripts/setup-wsl-portable-build-deps.sh --self-test + # Runs here, on the GitHub-hosted lane, rather than on the GPU runners the + # reclaim protects: the matching rules are what regress, and they are + # provable against decoy processes with no GPU present. + - name: GPU reclaim self-test + if: needs.changes.outputs.heavy == 'true' + # Twice, because one of the rules is conditional on the environment. + # The scenario root follows TMPDIR, and the self-test can only assert + # the redirected form when the shell running it actually has one — a + # hosted runner does not, so a single plain run reports that arm as + # "not exercised" and the nightly Strix lane's redirect would be the + # first place a break showed up. The second run supplies the redirect. + run: | + bash scripts/reclaim-gpu.sh --self-test + mkdir -p "${RUNNER_TEMP}/redirected-tmp" + TMPDIR="${RUNNER_TEMP}/redirected-tmp" bash scripts/reclaim-gpu.sh --self-test + - name: Release readiness self-test if: needs.changes.outputs.heavy == 'true' run: python scripts/release_readiness.py --self-test diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index b44bf797e1..f30cfcc408 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -177,25 +177,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Reclaim the GPU before running: a serve leaked by a killed/timed-out prior - # run (its Drop teardown never executed) can keep an engine process spinning - # on the GPU, starving this job's serves until it hits the timeout. Kill only - # e2e leftovers — scoped to /tmp/rocm-e2e-* and the e2e-target/e2e-shared - # trees — never the runner or any /workload manual-testing processes. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - # NOTE: no unanchored `pkill -f 'vulkan/llama-server'` — the lemonade - # Vulkan assistant an e2e scenario spawns lives under /tmp/rocm-e2e-* - # and is already caught by the first line; an unanchored pattern would - # also kill a legitimate /workload manual-testing serve on this shared - # self-hosted runner. - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the # GPU is missing, the driver is wedged, or VRAM is still saturated by a @@ -235,6 +224,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false — on this self-hosted runner we persist the build cache @@ -409,14 +402,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: bounded wait so a missing/wedged/held GPU fails fast (~90s) # instead of hanging to the job cap. Transient contention (VRAM still @@ -557,6 +550,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # Pool hosts don't preinstall cargo/rustup; bootstrap ourselves with @@ -673,10 +670,33 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaimed" + # Mirrors the SHAPE of scripts/reclaim-gpu.sh's rule — an E2E-owned root + # AND an engine process — but is not equivalent to it. Native Windows has + # no bash, so the rule is restated here rather than shared. Keep the root + # alternation in step with the script's E2E_ROOTS (`e2e-prewarm` is the + # shared runtime tree the engine binaries actually run from); xtask's + # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. + # + # The engine half diverges in two pre-existing ways, tracked in EAI-8815. + # They fail in OPPOSITE directions, and only one can cost a bystander: + # - a MISS: `rocm ` (literal trailing space) cannot match Windows' + # `rocm.exe daemon`, so such a leak is not reclaimed here; + # - an OVER-MATCH: `__engine-serve-http` sits in the root alternation + # rather than the engine one, so on Windows a non-E2E managed serve + # satisfies the root half from that marker alone and can be killed. + # Do not read this as "same markers as the script" until that is fixed. + $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } + foreach ($p in $leaked) { + Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" + Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue + } + # SIGNALLED, not terminated, for the same reason the bash summary says + # so: -ErrorAction SilentlyContinue swallows a Stop-Process that could + # not kill its target, and this count is the SELECTED set, so a + # process that survived is still counted. Weaker than the bash line, + # which at least counts only what it re-verified before signalling. + Write-Host "reclaim: $(@($leaked).Count) process(es) signalled" # GPU preflight (best-effort on Windows): bounded wait for the GPU to be # available so a wedged/held GPU fails fast instead of hanging to the cap. @@ -1211,6 +1231,14 @@ jobs: exit 0 fi echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${pool} never dropped below the floor — a serve is likely still holding the GPU" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + # From the guest clone, like every sibling step: `wsl -- bash ` + # gets no --cd, so a bare relative path would resolve against whatever + # directory WSL maps the Windows step's cwd to, and `|| true` would + # hide a miss. + (cd /root/work/rocm-cli && bash scripts/reclaim-gpu.sh --report-holders) || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does @@ -1378,18 +1406,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pod is usually fresh (scale-to-zero recreates it per job), but the - # work PVC is not, and two jobs can share one pod's life. Same reclaim as - # e2e-gpu, scoped to e2e leftovers only. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait, as in e2e-gpu. The pod requests `amd.com/gpu: 1`, so # rocm-smi sees exactly the one card Kueue admitted it for. The floor is @@ -1420,6 +1444,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false for the same reason as e2e-gpu — the build cache is kept in @@ -1506,18 +1534,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pod is usually fresh (the runner scales to zero between jobs), but the - # work volume is not, and two jobs can share one pod's life. Same reclaim as - # e2e-gpu, scoped to e2e leftovers only. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait, as in e2e-gpu. The pod is admitted for exactly one card, so # rocm-smi sees that card alone. The floor matches the other Instinct lane @@ -1549,6 +1573,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false for the same reason as e2e-gpu — the build cache is kept in diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 67e6181147..dd89dda9e9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -370,18 +370,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Reclaim the GPU before running: kill only e2e leftovers scoped to - # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or - # any /workload manual-testing processes (see e2e-selfhosted.yml e2e-gpu for the same). + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the # GPU is missing, the driver is wedged, or VRAM is still saturated by a @@ -421,6 +417,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false — this self-hosted runner persists the build cache itself via @@ -494,15 +494,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU. Floor is half of the R9700S's 32GB — high # enough to catch a leftover serve, low enough that a clean pod passes on @@ -532,6 +531,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 @@ -585,15 +588,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU. Floor matches the per-PR MI350P lane and the # other Instinct lane — high enough to catch a leftover serve, low enough @@ -623,6 +625,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 @@ -694,12 +700,8 @@ jobs: - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs run: | mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "prepared + reclaimed" + echo "prepared" + bash scripts/reclaim-gpu.sh - name: GPU preflight (bounded wait for an available GPU) run: | @@ -831,6 +833,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - name: Ensure Rust toolchain @@ -887,10 +893,33 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaimed" + # Mirrors the SHAPE of scripts/reclaim-gpu.sh's rule — an E2E-owned root + # AND an engine process — but is not equivalent to it. Native Windows has + # no bash, so the rule is restated here rather than shared. Keep the root + # alternation in step with the script's E2E_ROOTS (`e2e-prewarm` is the + # shared runtime tree the engine binaries actually run from); xtask's + # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. + # + # The engine half diverges in two pre-existing ways, tracked in EAI-8815. + # They fail in OPPOSITE directions, and only one can cost a bystander: + # - a MISS: `rocm ` (literal trailing space) cannot match Windows' + # `rocm.exe daemon`, so such a leak is not reclaimed here; + # - an OVER-MATCH: `__engine-serve-http` sits in the root alternation + # rather than the engine one, so on Windows a non-E2E managed serve + # satisfies the root half from that marker alone and can be killed. + # Do not read this as "same markers as the script" until that is fixed. + $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } + foreach ($p in $leaked) { + Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" + Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue + } + # SIGNALLED, not terminated, for the same reason the bash summary says + # so: -ErrorAction SilentlyContinue swallows a Stop-Process that could + # not kill its target, and this count is the SELECTED set, so a + # process that survived is still counted. Weaker than the bash line, + # which at least counts only what it re-verified before signalling. + Write-Host "reclaim: $(@($leaked).Count) process(es) signalled" # Best-effort when rocm-smi is absent; when present, fail within the bounded # ceiling instead of letting a held or wedged GPU consume the full job cap. @@ -1329,6 +1358,14 @@ jobs: exit 0 fi echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${pool} never dropped below the floor — a serve is likely still holding the GPU" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + # From the guest clone, like every sibling step: `wsl -- bash ` + # gets no --cd, so a bare relative path would resolve against whatever + # directory WSL maps the Windows step's cwd to, and `|| true` would + # hide a miss. + (cd /root/work/rocm-cli && bash scripts/reclaim-gpu.sh --report-holders) || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does diff --git a/docs/testing.md b/docs/testing.md index 22d3524d6e..1586167c1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -835,6 +835,7 @@ cargo test -p rocm-core model_recipe_index_signature_accepts_generated_key_and_r cargo test -p rocm --bin rocm wsl_rocdxg_generated_digest_step_accepts_only_the_matching_file python scripts/release_readiness.py --self-test bash scripts/setup-wsl-portable-build-deps.sh --self-test +bash scripts/reclaim-gpu.sh --self-test ``` The ROCDXG digest check pins the shell fragment the WSL driver plan generates @@ -847,6 +848,21 @@ that is then installed as root, and this replaces the self-test that shipped with the removed `scripts/wsl_setup_rocdxg.sh`. It needs a POSIX shell and `sha256sum`, so it is Unix-only. +`reclaim-gpu.sh` frees the GPU from engine processes a killed prior run leaked. +The self-hosted Linux GPU lanes run it before their GPU preflight; the two +native Windows lanes have no bash and restate the rule in PowerShell instead, +with their root list pinned to the script's by an `xtask` contract test. The +WSL lanes skip the proactive reclaim before preflight — they provision a fresh +guest per job and unregister it afterwards, so no prior run's processes survive +into them — but they still call the script for `--report-holders` when their +preflight fails, as the other bash lanes do. The two native Windows lanes call +it at neither point: their preflight failure path has no PowerShell equivalent +of the diagnostic. Its `--self-test` +needs no GPU — it spawns decoy processes and asserts which ones the matching +rule selects — so it runs on the GitHub-hosted lane rather than only where GPU +hardware is held, under the same `heavy` path filter as the other checks there +(which any change under `scripts/` reaches). + The release-readiness self-test is cross-platform and uses only workspace-local temporary files under `.rocm-work/tests/release-readiness`. It also checks exact release asset sets, so stale archives and orphan checksum/signature sidecars in diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh new file mode 100755 index 0000000000..e5172667ac --- /dev/null +++ b/scripts/reclaim-gpu.sh @@ -0,0 +1,1269 @@ +#!/usr/bin/env bash +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +# Reclaim the GPU from E2E processes leaked by a killed or timed-out prior run, +# whose Drop teardown never executed. +# +# WHY A SCRIPT AND NOT INLINE `pkill -f`: the inline version anchored each +# pattern to a scenario temp dir (`/tmp/rocm-e2e`) or the shared *cache* +# (`e2e-shared`), but a managed serve runs the engine binary out of the shared +# *pre-warm runtime* tree — `$RUNNER_WORKSPACE/e2e-prewarm-*/data/runtimes/...` +# — which matched neither. `pkill -f` also matches the whole command line in +# order, and argv[0] is the binary path, so a `/tmp/rocm-e2e-*` model argument +# appearing later could not rescue those patterns either. A leaked llama-server +# was therefore invisible to reclaim and held the card until the runner itself +# was replaced, failing every later job on that runner at the GPU preflight. +# +# That is why a miss here is expensive rather than merely untidy: on the +# container lanes the pod is usually fresh per job, but the work volume is not +# and two jobs can share one pod's life — so a single leak the reclaim cannot +# match poisons every job scheduled to that runner until the pod is recreated. +# +# The rule here is order-independent and covers every E2E-owned tree: a process +# is reclaimed when its command line names an E2E root AND an engine/serve +# process. Both halves are required, which is what spares a legitimate +# `/workload` manual-testing serve on a shared self-hosted runner in the +# ordinary case: it names no E2E root, so the root half does not hold. +# +# That is a narrowing, NOT a guarantee, and it is stated here rather than only +# in WHAT THE ANCHORING DOES AND DOES NOT BUY below, because a reader who stops +# at this paragraph is the operator deciding whether their own serve is safe. A +# `/workload` serve IS selected when its command line happens to name a root in +# an argument — `--model-path /home/dev/e2e-shared-models/x.gguf` is the +# measured case. Read that section before relying on this one. + +set -euo pipefail + +# E2E-owned roots. A process must name one of these to be considered ours. +# /tmp/rocm-e2e per-scenario temp dirs (plus the TMPDIR-derived form below) +# e2e-shared E2E_SHARED_CACHE_DIR (models, HF weights) +# e2e-prewarm E2E_SHARED_RUNTIMES_DIR — the pre-warmed runtime tree the +# engine binaries actually live in +# e2e-target CARGO_TARGET_DIR for the suite's own binaries +# +# The scenario root stays ABSOLUTE, and follows TMPDIR. +# +# Scenario dirs come from `TempDir::with_prefix("rocm-e2e-")`, which builds on +# `std::env::temp_dir()` and so follows TMPDIR; the nightly Strix lane redirects +# it to `$HOME/actions-runner/tmp`. A bare `/tmp/rocm-e2e` kept matching there +# only by accident — that directory is itself named `tmp`, so the substring +# still appeared — and would silently stop matching under any redirect not +# named `tmp`. Deriving the redirected form fixes that for ANY target. +# +# WHAT THE ANCHORING DOES AND DOES NOT BUY. Roots are matched as unanchored +# substrings of the WHOLE command line, so any root also matches a process that +# merely mentions it in an ARGUMENT. The bare segment `rocm-e2e` — what both +# PowerShell mirrors use — makes that trivially reachable: a hand-run +# `--hf-repo myorg/rocm-e2e-baseline-7b` on a shared runner is selected and +# SIGKILLed. Measured, and briefly shipped, which is why it is spelled out. +# +# Anchoring the scenario root to an absolute path NARROWS that surface. It does +# not close it, and the `/workload` narrowing stated at the top of this file is +# correspondingly weaker than it would read on its own: +# +# - under a redirected TMPDIR, an argument naming a sibling path still +# matches — `--extra-data-dir /tmp/mydir/rocm-e2e-notes` with +# TMPDIR=/tmp/mydir; +# - the three segment roots are NOT anchored at all, and "shared" and "target" +# are ordinary words. `--model-path /home/dev/e2e-shared-models/x.gguf` and +# `--served-model-name e2e-target-vs-baseline` are both selected today. +# +# Both predate this script and neither is fixed here; closing them properly +# means matching on a path boundary rather than a substring, across the bash +# rule and both mirrors together. Stated rather than implied, because an earlier +# revision of this comment claimed these roots "name directories no argument +# plausibly carries" — which is false, and is the same overclaim that made the +# bare-segment change look safe. +E2E_ROOTS=( + '/tmp/rocm-e2e' + 'e2e-shared' + 'e2e-prewarm' + 'e2e-target' +) +# Appended rather than listed, because it is only known at run time. Skipped +# when TMPDIR is unset or already /tmp, so the list stays exactly the pinned one +# on every lane that does not redirect. A RELATIVE TMPDIR is skipped too: it +# would append a relative root and quietly give up the anchoring above, and no +# lane sets one. +if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + E2E_ROOTS+=("${TMPDIR%/}/rocm-e2e") +fi + +# Engine/serve processes that can hold VRAM. Matched anywhere in the command +# line, so a wrapper or an absolute binary path both work. +ENGINE_MARKERS=( + 'llama-server' + 'vllm' + '__engine-serve-http' + 'rocm daemon' +) + +# Seconds to wait for a TERM'd process to exit before escalating to KILL. +TERM_GRACE_SECS="${RECLAIM_TERM_GRACE_SECS:-5}" + +# Set by --self-test so its decoys are the only things in scope. Never set in +# CI: an empty scope means "any process matching the rules above". +SELFTEST_SCOPE="${RECLAIM_SELFTEST_SCOPE:-}" + +usage() { + cat <<'EOF' +Usage: reclaim-gpu.sh [--dry-run | --report-holders | --self-test] + + (no flags) Terminate leaked E2E engine processes and report what was killed. + --dry-run List what would be terminated; kill nothing. + --report-holders Print current GPU holders and candidate processes, for + diagnosing a preflight failure. Kills nothing, never fails. + --self-test Verify the matching rules against decoy processes. No GPU needed. +EOF +} + +# True while a pid is a live process. A zombie is NOT alive: it has already +# released its VRAM and is only waiting to be reaped, but `kill -0` still +# succeeds on it, which would read as "ignored SIGTERM". +process_alive() { + local pid="$1" + local state + # /proc//stat field 3 is the state, but field 2 (comm) may contain + # spaces, so cut after the closing paren rather than counting fields. + state="$(sed 's/.*) //' "/proc/${pid}/stat" 2>/dev/null | cut -d' ' -f1)" || return 1 + [[ -n "${state}" ]] || return 1 + [[ "${state}" != "Z" ]] +} + +# Read a process's command line, NUL-separated in /proc, as a single string. +# Fails when the process is gone or its command line is empty (a kernel thread, +# or a zombie whose argv has already been released). +# +# NEWLINE is flattened along with the NUL separators the kernel inserted, +# because it is the record terminator `select_leaked` emits and `reclaim` reads +# back. A newline INSIDE an argv element is content, not a separator, and +# `--chat-template` or `--prompt` can carry one. Left intact, such a process +# emits a record that splits into two on the way back: the real pid arrives +# carrying a truncated command line, fails the identity check against its own +# full one, and is reported "recycled ... not signalling" — so the leak survives +# both TERM and KILL while the summary prints "0 process(es) signalled". That is +# the silent miss this whole script exists to end, so it is fixed at the single +# point every consumer already reads through rather than at each of them. +# +# The TAB is deliberately NOT flattened, though it is the field separator. It +# does not need to be, measured rather than reasoned: +# +# record "\t/path/llama-server --tmpl a\tb " +# read -r pid cmdline -> pid=[] cmdline=[/path/llama-server --tmpl a\tb ] +# +# `read` gives leftover words AND their intervening separators to the LAST name, +# so a tab EMBEDDED in the command line lands in `cmdline` verbatim. Separators +# are stripped only where the assigned field BEGINS or ENDS with them, and BOTH +# edges are ruled out here — each by its own property, so both are named: +# +# - leading: the command line starts with argv[0], a path, so the trailing +# field never begins with a separator. (Were it to, a leading tab WOULD be +# stripped: `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`.) +# - trailing: /proc//cmdline ends in a NUL for every process as execve +# left it — every engine this rule matches — and flattening that NUL +# leaves a trailing space, so a tab is never the last byte. This one is +# load-bearing and measured — with the trailing space `…arg\t ` survives +# intact, without it the same tab is stripped. The same property is relied +# on again at the self-test's record check; if it ever changes, both move +# together. +# +# Not every process, though. One that rewrites its argv AND environment +# with no NUL left in either (setproctitle taken to the limit) gets a +# cmdline with no terminator at all — measured, it can end in a tab. The +# tab is then stripped, the identity check below fails, and the pid is +# reported "recycled" and not signalled: a miss, never a wrong kill. +# +# An earlier revision also claimed the tab "shifts the field boundary" and +# flattened it for that reason — also false, and deleting the tab from the +# flattening set failed no check in this file. Flattening it anyway would have +# coarsened the identity comparison below for nothing. +# +# Matching is unaffected either way: no root or marker contains a newline. The +# identity comparison in same_selected_process is coarsened only for the newline +# — two command lines differing ONLY in a newline-versus-space at one position +# now read as equal. Both sides come through here, so they are flattened alike. +# That is a real cost, weighed against a silent miss any engine invocation +# carrying a prompt can reach. +# +# Redirect stderr BEFORE the input redirection: the shell applies them left to +# right, so `/dev/null` still lets the shell's own "No such file" reach +# the terminal when the open fails. +cmdline_of() { + local cmdline + cmdline="$(tr '\0\n' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 + [[ -n "${cmdline}" ]] || return 1 + printf '%s' "${cmdline}" +} + +# True when a command line names an E2E root AND an engine marker. +# +# Split out of select_leaked so the self-test can drive every root and every +# marker directly. Exercising them through spawned processes would need one +# process per list entry; leaving them undriven is how `e2e-shared`, `vllm`, +# `__engine-serve-http` and `rocm daemon` came to be removable with the +# self-test still green. +cmdline_matches_rule() { + local cmdline="$1" + local root engine + local has_root=0 + local has_engine=0 + for root in "${E2E_ROOTS[@]}"; do + case "${cmdline}" in + *"${root}"*) + has_root=1 + break + ;; + *) ;; + esac + done + [[ "${has_root}" == 1 ]] || return 1 + for engine in "${ENGINE_MARKERS[@]}"; do + case "${cmdline}" in + *"${engine}"*) + has_engine=1 + break + ;; + *) ;; + esac + done + [[ "${has_engine}" == 1 ]] +} + +# Whether is still running the command line it was selected with. +# 0 same process +# 1 a DIFFERENT command line — the pid was recycled +# 2 gone: exited between the liveness check and this read +# The 1/2 split matters because they are not the same event, and reporting a +# process that simply exited as "recycled" misdescribes a benign race. +same_selected_process() { + local pid="$1" + local expected="$2" + local current + current="$(cmdline_of "${pid}")" || return 2 + [[ "${current}" == "${expected}" ]] +} + +# Print "pidcommand line" for every process whose command line names both +# an E2E root and an engine marker. +select_leaked() { + local cmdline_file pid cmdline + for cmdline_file in /proc/[0-9]*/cmdline; do + pid="${cmdline_file#/proc/}" + pid="${pid%/cmdline}" + # Never reclaim ourselves or our own shell. + if [[ "${pid}" == "$$" || "${pid}" == "${PPID}" ]]; then + continue + fi + # A process can exit between the glob and the read; that is not an error. + cmdline="$(cmdline_of "${pid}")" || continue + + if [[ -n "${SELFTEST_SCOPE}" ]]; then + case "${cmdline}" in + *"${SELFTEST_SCOPE}"*) ;; + *) continue ;; + esac + fi + + cmdline_matches_rule "${cmdline}" || continue + + printf '%s\t%s\n' "${pid}" "${cmdline}" + done +} + +# TERM, wait out the grace period, then KILL whatever is left. Reports every +# process it acts on: an unconditional "reclaimed" tells the next reader +# nothing, and a silent no-op is how the original defect stayed hidden. +reclaim() { + local dry_run="$1" + local selected pid cmdline rc + local killed=0 + local waited=0 + + selected="$(select_leaked)" + if [[ -z "${selected}" ]]; then + echo "reclaim: no leaked E2E engine processes found" + return 0 + fi + + while IFS=$'\t' read -r pid cmdline; do + [[ -n "${pid}" ]] || continue + if [[ "${dry_run}" == 1 ]]; then + echo "reclaim: would terminate pid=${pid} cmd=${cmdline}" + else + # Same guard as the escalation below, and for the same reason. The window + # is shorter here — no grace period — but it is not zero: select_leaked + # walks the whole of /proc, so a pid can be freed and reissued between + # the scan that selected it and this signal. A bystander is just as dead + # from TERM as from KILL, so the check cannot be reserved for the latter. + rc=0 + same_selected_process "${pid}" "${cmdline}" || rc=$? + case "${rc}" in + 0) ;; + 2) + echo "reclaim: pid=${pid} exited before it could be terminated" + continue + ;; + *) + echo "reclaim: pid=${pid} was recycled before it could be terminated, not signalling" + continue + ;; + esac + echo "reclaim: terminating pid=${pid} cmd=${cmdline}" + kill -TERM "${pid}" 2>/dev/null || true + fi + killed=$((killed + 1)) + done <<<"${selected}" + + if [[ "${dry_run}" == 1 ]]; then + echo "reclaim: ${killed} process(es) would be terminated (dry run)" + return 0 + fi + + while [[ "${waited}" -lt "${TERM_GRACE_SECS}" ]]; do + if [[ -z "$(select_leaked)" ]]; then + break + fi + sleep 1 + waited=$((waited + 1)) + done + + # Anything still alive after the grace period gets SIGKILL. VRAM is released + # by the kernel when the process dies, so this is what actually frees the card. + while IFS=$'\t' read -r pid cmdline; do + [[ -n "${pid}" ]] || continue + process_alive "${pid}" || continue + # This loop walks the pre-TERM snapshot, and a pid freed during the grace + # window can be handed to an unrelated process. Escalating on the pid alone + # would SIGKILL that bystander, so require the command line to still be the + # one we selected. Not killing a genuine holder is recoverable — the next + # job's reclaim sees it again — where killing a bystander is not. + rc=0 + same_selected_process "${pid}" "${cmdline}" || rc=$? + case "${rc}" in + 0) + echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" + kill -KILL "${pid}" 2>/dev/null || true + ;; + 2) echo "reclaim: pid=${pid} exited during the grace period" ;; + *) echo "reclaim: pid=${pid} was recycled during the grace period, not escalating" ;; + esac + done <<<"${selected}" + + # SIGNALLED, not terminated. The count is incremented where TERM is sent, and + # both kill calls are `2>/dev/null || true` so a signal this process was not + # permitted to send is invisible here; the SIGKILL loop does not touch the + # count at all. Claiming termination would therefore report success for a + # process still holding the card — the exact failure mode the unconditional + # `reclaimed` line had, which is what let this go unnoticed for hours. The + # per-process lines above are where the detail lives. + echo "reclaim: ${killed} process(es) signalled" +} + +# Diagnostics for a preflight that hit its ceiling. Never fails: it runs on the +# failure path, where masking the real error would be worse than missing output. +report_holders() { + local engine + local -a marker_args=() + echo "--- rocm-smi KFD processes (per-process VRAM) ---" + # Process names show as UNKNOWN inside a container: KFD reports host PIDs, + # which do not resolve in the container's PID namespace. The VRAM column is + # still the answer to "what is holding the card". + timeout 15 rocm-smi --showpids 2>&1 | head -40 || true + echo "--- engine/serve processes visible here ---" + # Derived from ENGINE_MARKERS rather than spelled out again. A second copy of + # the list is exactly the drift the PowerShell mirrors demonstrated and that + # the xtask contract test exists to catch — and this call site had no such + # guard, so a marker added above would have quietly stopped appearing here. + # Deriving it needs no guard: there is no longer a copy that can go stale. + # + # -F, not an -E alternation: the markers are literal substrings everywhere + # else in this script, and building a regex out of them would give a future + # marker containing a metacharacter a different meaning here than in the rule. + # + # No empty-array guard. Given zero patterns grep prints its usage to stderr + # and exits 2; the `|| true` below swallows the STATUS, so the section would + # carry that usage error in place of any holders — wrong, but not silent. + # Reaching it needs ENGINE_MARKERS to be empty, which + # assert_rule_covers_every_list_entry already fails on: it pins the list to + # its four entries by exact comparison, so an emptied array reds the + # self-test before anything gets here. Guarding it again would add a branch + # the self-test could never reach, which is worth less than the assertion + # that already covers it. + for engine in "${ENGINE_MARKERS[@]}"; do + marker_args+=(-e "${engine}") + done + # shellcheck disable=SC2009 # pgrep cannot print elapsed time, and how long a + # holder has been alive is what distinguishes a leak from this job's own serve. + ps -eo pid,etimes,args 2>/dev/null | + grep -Fi "${marker_args[@]}" | + grep -v grep | + head -40 || true + echo "--- of those, E2E-owned (reclaim would take these) ---" + select_leaked || true +} + +# Spawn a decoy whose argv[0] is a real-shaped path, so the match is tested +# against the same string a live engine process would present. +spawn_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + cp /bin/sleep "${path}" + # Detach stdio: this runs inside a command substitution, and a background + # child holding the capture pipe open would block the caller until it exits. + "${path}" 300 >/dev/null 2>&1 & + echo $! +} + +# Spawn a process that becomes a ZOMBIE and stays one, plus the keeper holding +# it in that state. Echoes " "; the zombie pid is the +# literal string `none` if one could not be produced — NOT an empty field, for +# the reason given at the sentinel itself. Test `!= "none"`, never `-z`. +# +# A zombie is the second way into cmdline_of's failure path and the only one +# that reaches its emptiness check: /proc//cmdline still OPENS for a +# zombie, it just reads zero bytes, because the kernel has released the argv +# while the process table entry remains. +spawn_zombie() { + local dir="$1" + local keeper_pid zombie_pid state + local waited=0 + mkdir -p "${dir}" + # The subshell starts a short-lived child and then `exec`s a long sleep, so + # the parent that would reap it is replaced by a process that never calls + # wait(). The child therefore stays a zombie for as long as the keeper lives. + # Bash reaps its OWN background children, which is why this needs the exec. + ( + sleep 0.1 & + echo $! >"${dir}/zombie.pid" + exec sleep 300 + ) >/dev/null 2>&1 & + keeper_pid=$! + # Wait for the child to actually reach state Z rather than assuming it has. + while [[ "${waited}" -lt 50 ]]; do + if [[ -s "${dir}/zombie.pid" ]]; then + zombie_pid="$(cat "${dir}/zombie.pid")" + state="$(sed 's/.*) //' "/proc/${zombie_pid}/stat" 2>/dev/null | cut -d' ' -f1)" || state='' + [[ "${state}" == "Z" ]] && break + fi + sleep 0.1 + waited=$((waited + 1)) + done + # Sentinel, not an empty field: `read` skips leading whitespace, so echoing an + # empty first field would shift the KEEPER's pid into the caller's zombie_pid + # and make the "could not produce one" branch unreachable. + [[ "${state:-}" == "Z" ]] || zombie_pid='none' + echo "${zombie_pid} ${keeper_pid}" +} + +# Spawn a decoy that IGNORES SIGTERM, so the TERM -> grace -> KILL escalation is +# exercised. A plain `cp /bin/sleep` decoy dies on the first TERM and leaves the +# escalation branch unreached, which is how it went untested. +spawn_stubborn_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + # The foreground `sleep 1` children are short-lived and name no E2E root, so + # they are never selected and leave nothing behind once the parent is killed. + cat >"${path}" <<'DECOY' +#!/usr/bin/env bash +trap '' TERM +while :; do sleep 1; done +DECOY + chmod +x "${path}" + "${path}" >/dev/null 2>&1 & + echo $! +} + +# Spawn a decoy carrying the record delimiters INSIDE one argument — a newline +# and a tab, as a `--chat-template` or `--prompt` value does. +# +# Not spawn_decoy with an extra argument: that decoy is `cp /bin/sleep`, and +# sleep rejects a non-numeric argument and exits before it can be observed. This +# needs something that ignores what it is handed and stays alive. +# +# Ordinary in every other respect — it must die on SIGTERM, so it is the stubborn +# decoy's body without the trap. Its `sleep 1` children name no E2E root, so they +# are never selected, and none outlives the scratch tree by more than a second. +spawn_delimiter_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + cat >"${path}" <<'DECOY' +#!/usr/bin/env bash +while :; do sleep 1; done +DECOY + chmod +x "${path}" + "${path}" $'--chat-template\nrole:\tuser' >/dev/null 2>&1 & + echo $! +} + +# Every root and every marker, asserted individually against the rule. +# +# The expectation is written out rather than derived from the arrays: a loop +# over E2E_ROOTS cannot notice a root DELETED from E2E_ROOTS, which is exactly +# the drift that left `e2e-shared`, `vllm`, `__engine-serve-http` and +# `rocm daemon` removable with the self-test green. The duplication is the +# point — changing either list must be a deliberate edit in two places, one of +# which names the PowerShell mirrors that also have to move. +assert_rule_covers_every_list_entry() { + local failures=0 + local root engine + local expected_roots=('/tmp/rocm-e2e' 'e2e-shared' 'e2e-prewarm' 'e2e-target') + local expected_markers=('llama-server' 'vllm' '__engine-serve-http' 'rocm daemon') + # The TMPDIR-derived scenario root is appended at load time, so the pinned + # list is the STATIC one plus that entry when it applies. Computed the same + # way the script does rather than assumed absent: this function also runs on a + # developer machine, and macOS sets TMPDIR for every shell. + if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + expected_roots+=("${TMPDIR%/}/rocm-e2e") + fi + + if [[ "$(printf '%s\n' "${E2E_ROOTS[@]}")" != "$(printf '%s\n' "${expected_roots[@]}")" ]]; then + echo "FAIL: E2E_ROOTS changed — update this expectation AND both PowerShell mirrors" + echo " script: ${E2E_ROOTS[*]}" + echo " expected: ${expected_roots[*]}" + failures=$((failures + 1)) + fi + if [[ "$(printf '%s\n' "${ENGINE_MARKERS[@]}")" != "$(printf '%s\n' "${expected_markers[@]}")" ]]; then + echo "FAIL: ENGINE_MARKERS changed — update this expectation AND both PowerShell mirrors" + echo " script: ${ENGINE_MARKERS[*]}" + echo " expected: ${expected_markers[*]}" + failures=$((failures + 1)) + fi + + for root in "${expected_roots[@]}"; do + if ! cmdline_matches_rule "${root}/bin/llama-server --model m"; then + echo "FAIL: root '${root}' with an engine marker does not match the rule" + failures=$((failures + 1)) + fi + if cmdline_matches_rule "${root}/bin/e2e-harness --exact"; then + echo "FAIL: root '${root}' matched with NO engine marker; the AND rule is broken" + failures=$((failures + 1)) + fi + done + + for engine in "${expected_markers[@]}"; do + if ! cmdline_matches_rule "e2e-prewarm/bin/${engine} --serve"; then + echo "FAIL: marker '${engine}' under an E2E root does not match the rule" + failures=$((failures + 1)) + fi + if cmdline_matches_rule "workload/bin/${engine} --serve"; then + echo "FAIL: marker '${engine}' matched with NO E2E root; a manual serve is not safe" + failures=$((failures + 1)) + fi + done + + # The scenario root must stay ANCHORED to an absolute path. These are argument + # strings, not paths: a hand-run serve on a shared runner that merely NAMES an + # E2E-suite baseline. A bare `rocm-e2e` root matches both and SIGKILLs them. + # The `/workload` fixtures above cannot catch it — they carry no arguments at + # all, so they stay green against exactly this mistake. It was made, and this + # is what would have caught it. + # + # This pins the scenario root ONLY. The header explains why that is a + # narrowing rather than a fix: the three segment roots stay unanchored, so + # `--model-path /home/dev/e2e-shared-models/x.gguf` is still selected. No + # assertion here claims otherwise, deliberately — a green run means the + # scenario root did not regress, not that a manual serve is safe. + for root in '--hf-repo myorg/rocm-e2e-baseline-7b' '--model-alias my-rocm-e2e-comparison'; do + if cmdline_matches_rule "/workload/manual/llama-server ${root}"; then + echo "FAIL: a manual serve was selected for merely NAMING '${root}'; the scenario root is not anchored" + failures=$((failures + 1)) + fi + done + + # The other half of the same decision: anchoring must not cost the redirect. + # Only assertable when this shell actually has a redirected TMPDIR, so it says + # which case ran rather than reporting a pass for a branch it skipped. + if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + if cmdline_matches_rule "${TMPDIR%/}/rocm-e2e-ab12/bin/llama-server --model m"; then + echo "ok: a scenario tree under the redirected TMPDIR is selected" + else + echo "FAIL: TMPDIR is redirected but a scenario tree under it is not selected" + failures=$((failures + 1)) + fi + else + echo "note: no absolute TMPDIR redirect in this shell, so that arm was not exercised" + fi + + if [[ "${failures}" -eq 0 ]]; then + echo "ok: every E2E root and every engine marker is individually enforced" + fi + return "${failures}" +} + +# Decoys the self-test has spawned so far, read by its EXIT trap. +# +# Global, and read at trap FIRE time rather than expanded at definition time the +# way the scratch-dir paths are: a pid is not known until its spawn returns, so +# the one thing that must not be baked into the trap string is this list. +SELFTEST_DECOY_PIDS=() + +selftest_track_decoy() { + SELFTEST_DECOY_PIDS+=("$1") +} + +# SIGKILL, not TERM: one decoy ignores TERM by design, and this runs on the way +# out with nothing left to wait for it. +selftest_kill_decoys() { + [[ "${#SELFTEST_DECOY_PIDS[@]}" -gt 0 ]] || return 0 + kill -KILL "${SELFTEST_DECOY_PIDS[@]}" 2>/dev/null || true +} + +self_test() { + local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy + local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out + local outside outside_decoy outside_pid outside_cmd + local delimiter_decoy delimiter_pid delimiter_raw record_pid + local forced_out guard_pid + local escapee_pid escapee_cmd guard_rc probe_cmd + local zombie_pid zombie_keeper_pid + local holders_out holders_engines tmp_case + local decoy_pids + local superseded_hit=0 + local probe_failures=0 + local containment_failures=0 + local fixture_failures=0 + local record_failures=0 + local list_failures=0 + local failures=0 + + assert_rule_covers_every_list_entry || list_failures=$? + failures=$((failures + list_failures)) + + # Deliberately NOT under /tmp/rocm-e2e: that prefix is one of the roots the + # old patterns did match, which would mask the regression this guards. + tmp="$(mktemp -d /tmp/reclaim-selftest-XXXXXX)" + # Armed the moment there is something to remove, and widened below once the + # second tree exists. The `mktemp` that follows can fail, and under `set -e` + # that aborts the function — with `tmp` already on disk and, if the trap were + # installed only afterwards, nothing left to clean it up. + # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time + trap "selftest_kill_decoys; rm -rf '${tmp}'" EXIT + # A SECOND tree, outside the scope, for the bystander decoy below. Everything + # reachable through SELFTEST_SCOPE is inside `tmp` by construction, so a + # negative case for the scope filter cannot live there. + outside="$(mktemp -d /tmp/reclaim-selftest-out-XXXXXX)" + # Widened in the statement immediately after the one that created it, for the + # same reason the narrow trap above exists: anything fallible in between is a + # window where a tree is on disk with nothing arranged to remove it. + # shellcheck disable=SC2064 # expand both paths now, at trap definition time + trap "selftest_kill_decoys; rm -rf '${tmp}' '${outside}'" EXIT + export RECLAIM_SELFTEST_SCOPE="${tmp}" + SELFTEST_SCOPE="${tmp}" + # The stubborn decoy never exits on its own, so the grace loop always runs to + # the ceiling. Keep it short: this is a unit-speed test, not a GPU lane. + TERM_GRACE_SECS=2 + + # The real shape: lemonade's engine binary inside the shared pre-warm runtime. + prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" + # A manual-testing serve on a shared runner: an engine, but no E2E root. + workload_decoy="${tmp}/workload/manual-serve/llama-server" + # An E2E root with NO engine marker — the suite's own test binary under + # CARGO_TARGET_DIR. This is what the `has_engine` half exists to spare, and + # without it in the fixtures that half can be deleted with the test still green. + harness_decoy="${tmp}/e2e-target/release/deps/e2e-harness" + # Same shape as the pre-warm decoy but ignores SIGTERM, so the escalation this + # script adds is reached. Without it, TERM alone ends every decoy and both the + # SIGKILL block and the `kill -TERM` call can be removed with the test green. + stubborn_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" + # Matches the rule in full — an E2E root AND an engine marker — but lies + # OUTSIDE the scope. It is the only fixture the SELFTEST_SCOPE filter can be + # observed doing anything to, and therefore the only reason check 7 can fail. + outside_decoy="${outside}/e2e-prewarm-bystander/bin/llama-server" + # Same pre-warm shape again, but its ARGUMENTS carry a newline and a tab — the + # two bytes the pidcmdline record format is built out of. Without + # a fixture whose command line contains them, the format is only ever exercised + # on strings that cannot break it. + delimiter_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" + + # Each pid is registered with the EXIT trap the moment it exists, for the same + # reason the scratch-dir traps above are widened one statement at a time: + # registering them only in the list further down would leave every + # ALREADY-spawned decoy running with nothing arranged to kill it — including + # the stubborn one, which ignores SIGTERM and loops forever. Measured: a + # failing statement between two spawns left 3 survivors before this change and + # 0 after. + # + # NOT because `set -e` aborts on a failing spawn — it does not, and an earlier + # version of this comment said it did. `inherit_errexit` is off, so errexit + # does not reach inside `$( )`, and each helper ends in an `echo` that + # succeeds regardless; a hard failure injected into a spawn helper runs the + # suite to a green exit 0. The routes that DO reach the trap mid-way are a + # signal (a cancelled CI job), and any fallible statement between the spawns + # here — including one a later edit adds, which is the case this guards. + # + # The flip side is worth knowing: because a broken spawn does not abort, it + # yields a live-looking pid for a process that died immediately, and surfaces + # later as a confusing assertion failure rather than as "the spawn failed". + prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" + selftest_track_decoy "${prewarm_pid}" + workload_pid="$(spawn_decoy "${workload_decoy}")" + selftest_track_decoy "${workload_pid}" + harness_pid="$(spawn_decoy "${harness_decoy}")" + selftest_track_decoy "${harness_pid}" + stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" + selftest_track_decoy "${stubborn_pid}" + outside_pid="$(spawn_decoy "${outside_decoy}")" + selftest_track_decoy "${outside_pid}" + delimiter_pid="$(spawn_delimiter_decoy "${delimiter_decoy}")" + selftest_track_decoy "${delimiter_pid}" + read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" + selftest_track_decoy "${zombie_keeper_pid}" + # Every process this function spawned that can still be signalled, so cleanup + # is one list rather than a line kept in step at each early return. The zombie + # itself is absent deliberately: it is already dead, and killing its keeper is + # what lets init reap it. Same set the trap holds, by construction. + decoy_pids=("${SELFTEST_DECOY_PIDS[@]}") + # The registration itself, pinned. Without this, dropping a + # `selftest_track_decoy` call is silent AND leaks for real: both the EXIT trap + # and the final kill read this same list, so an untracked decoy is signalled + # by nothing. It survives even a GREEN run — measured, the `/workload` fixture + # outlives the suite by its full 300s, because `reclaim` is supposed to spare + # that one and the tracked list is the only other thing that would kill it. + # (An earlier version of this comment claimed such a decoy "is still killed at + # the end". It is not, and it is only incidentally true for the three fixtures + # `reclaim` itself signals.) + # + # LIMITATION, since an exact literal cannot carry its own maintenance: this + # catches a tracking call REMOVED from an existing decoy, where the count + # drops. It does NOT catch a decoy ADDED without one — the count stays at the + # stale literal and the run passes. Adding a spawn means bumping this number + # in the same edit; there is no mechanism enforcing that. + if [[ "${#SELFTEST_DECOY_PIDS[@]}" -ne 7 ]]; then + echo "FAIL: ${#SELFTEST_DECOY_PIDS[@]} decoys registered with the EXIT trap, expected 7" + echo " tracked: ${SELFTEST_DECOY_PIDS[*]}" + echo " a spawn is not being tracked; it would outlive the run, aborted or not" + failures=$((failures + 1)) + fi + # Give the decoys a moment to appear in /proc with their full argv. + sleep 1 + + # 1. Regression guard: the patterns this script replaced could not see a + # pre-warm engine process. If this ever matches, the decoy stopped being + # representative and the rest of the self-test proves nothing. + # + # Asked of the decoys' own command lines, not of the machine. The obvious + # `pgrep -f ''` also matches the shell that INVOKED it whenever + # that shell's own command line contains the pattern, so it reports a + # match with no such process alive; on a shared self-hosted runner an + # unrelated process would fail it too. Both are false FAILs in the check + # whose entire job is to say "the decoy is unrepresentative". + for probe_cmd in "$(cmdline_of "${prewarm_pid}")" "$(cmdline_of "${stubborn_pid}")"; do + # An unreadable probe would leave every pattern unmatched and print "ok" + # having tested nothing. Same reason the zombie arm below fails loudly + # rather than skipping: a check that cannot run must not report a pass. + if [[ -z "${probe_cmd}" ]]; then + echo "FAIL: could not read a decoy's command line; the regression guard tested nothing" + probe_failures=$((probe_failures + 1)) + continue + fi + if [[ "${probe_cmd}" =~ /tmp/rocm-e2e.*llama-server ]] || + [[ "${probe_cmd}" =~ e2e-shared.*llama-server ]]; then + superseded_hit=1 + fi + done + failures=$((failures + probe_failures)) + if [[ "${superseded_hit}" == 1 ]]; then + echo "FAIL: superseded patterns matched the pre-warm decoy; decoy is unrepresentative" + failures=$((failures + 1)) + elif [[ "${probe_failures}" -eq 0 ]]; then + echo "ok: superseded patterns do not match a pre-warm engine process (the defect)" + fi + + # 2. The new rule selects it. + selected="$(select_leaked)" + if grep -q "^${prewarm_pid} " <<<"${selected}"; then + echo "ok: pre-warm engine process is selected" + else + echo "FAIL: pre-warm engine process was not selected" + failures=$((failures + 1)) + fi + + # 3. The record format survives a command line containing BOTH delimiter bytes. + # + # select_leaked emits "pidcmdline" and reclaim reads it back + # with `IFS=$'\t' read -r`. The two bytes are not symmetric, and this check + # is the only place that says so: + # + # - a NEWLINE in an argv element splits one record into two. The genuine pid + # then arrives with a TRUNCATED command line, fails the identity check + # against its own full one, and is passed over as "recycled": the leak + # survives TERM and KILL alike while the run signs off with "0 process(es) + # signalled". cmdline_of flattens it for that reason, and deleting that + # flattening fails this check. + # - a TAB does NOT, even though it is the field separator, because `read` + # hands leftover separators to the last name and the trailing space left + # by the final NUL keeps a tab from ever being stripped as a trailing one. + # cmdline_of deliberately leaves it alone; see the reasoning there. + # + # The tab fixture therefore guards the FORMAT, not the flattening: it is + # what fails if the record shape is ever changed to something a tab can + # break. Asserting it as though it pinned a flattening step would be a claim + # of coverage that no mutation can falsify, which is how this check read + # before — it announced the tab as individually asserted while deleting the + # tab from the flattening set left every check green. + # + # Placed HERE, ahead of containment, deliberately. A split record also + # trips check 7's escape loop — its continuation line names no scratch + # tree — which arms the gate and returns before checks 8-11 ever run. The + # run is red either way; what this adds is the true cause, printed before + # the one check 7 would otherwise report in its place. + # Read RAW, flattening only the kernel's NUL separators: cmdline_of flattens + # the newline, so asking it would report the fixture carries one no matter what + # the decoy was actually given. + # Counted apart from the record loop below, for the same reason check 7 keeps + # its empty-selection count separate: these say the FIXTURE is unusable, the + # loop says the FORMAT is broken, and the "ok:" line must not be able to + # affirm the second while the first has just been denied. + # stderr redirected BEFORE the input, for the reason cmdline_of spells out: + # the shell applies them left to right, so the input-first form lets its own + # "No such file" reach the terminal on a failed open — here, noise printed by + # the very branch that exists to report a dead fixture cleanly. + if ! delimiter_raw="$(tr '\0' ' ' 2>/dev/null <"/proc/${delimiter_pid}/cmdline")"; then + # Distinguished from "carries no delimiter": if the decoy died before this + # read, the byte assertions below would report a format problem for what is + # really a dead fixture, and send the next reader after the wrong thing. + echo "FAIL: delimiter decoy's command line could not be read; the fixture is gone, not malformed" + fixture_failures=$((fixture_failures + 1)) + delimiter_raw='' + else + case "${delimiter_raw}" in + *$'\n'*) ;; + *) + echo "FAIL: delimiter decoy's command line carries no newline; the record-format check is vacuous" + fixture_failures=$((fixture_failures + 1)) + ;; + esac + case "${delimiter_raw}" in + *$'\t'*) ;; + *) + echo "FAIL: delimiter decoy's command line carries no tab; the record format is no longer exercised against one" + fixture_failures=$((fixture_failures + 1)) + ;; + esac + fi + if ! grep -q "^${delimiter_pid} " <<<"${selected}"; then + # Selection is NOT what this check tests — and note it survives the + # unflattened form, because the `^pid` grep matches the first line of a + # split record. It is asserted only so the loop below cannot report a pass + # having never seen a command line with a delimiter in it. + echo "FAIL: delimiter decoy was not selected; the record-format check has no fixture" + fixture_failures=$((fixture_failures + 1)) + fi + failures=$((failures + fixture_failures)) + while IFS=$'\t' read -r record_pid _; do + [[ -n "${record_pid}" ]] || continue + if [[ ! "${record_pid}" =~ ^[0-9]+$ ]]; then + echo "FAIL: selection record does not begin with a pid: '${record_pid}'" + record_failures=$((record_failures + 1)) + fi + done <<<"${selected}" + failures=$((failures + record_failures)) + # Gated on the fixture too, not just on the record loop. A loop that examined + # a command line with no delimiter in it proves nothing, so affirming it here + # would print "every record survives delimiters" directly beneath a FAIL line + # saying there were none — evidence contradicting itself on the run someone is + # reading, which is the very thing check 7 is shaped to avoid. + if [[ "${record_failures}" -eq 0 && "${fixture_failures}" -eq 0 ]]; then + echo "ok: every selection record survives delimiters in a command line" + fi + + # Checks 4 and 5 are NEGATIVE assertions, and a negative assertion is + # satisfied trivially by a selection that is empty. Both therefore gate their + # affirmation on the selection being non-empty, the same way check 7 gates its + # containment line and for the same reason: checks 2 and 6 already fail such a + # run, so this is not a false green — but "ok: a manual serve is not selected" + # printed on a run where nothing was selected at all is evidence that misleads + # whoever is diagnosing it. + # + # Reported as SKIP rather than counted as a failure, so the run's cause stays + # the check that actually detected the breakage rather than these two + # restating it. + + # 4. A manual-testing serve is left alone (engine marker, but no E2E root). + if grep -q "^${workload_pid} " <<<"${selected}"; then + echo "FAIL: /workload manual serve was selected; reclaim must not touch it" + failures=$((failures + 1)) + elif [[ -z "${selected}" ]]; then + echo "SKIP: /workload manual serve was not selected, but the selection is EMPTY — this check proved nothing" + else + echo "ok: /workload manual serve is not selected" + fi + + # 5. Both halves are required: an E2E root alone must not select. Deleting the + # `has_engine` requirement makes exactly this check fail and nothing else. + if grep -q "^${harness_pid} " <<<"${selected}"; then + echo "FAIL: E2E test binary was selected; the engine half of the rule is not enforced" + failures=$((failures + 1)) + elif [[ -z "${selected}" ]]; then + echo "SKIP: the E2E test binary was not selected, but the selection is EMPTY — this check proved nothing" + else + echo "ok: an E2E root without an engine marker is not selected" + fi + + # 6. A process that ignores SIGTERM is still selected. + if grep -q "^${stubborn_pid} " <<<"${selected}"; then + echo "ok: SIGTERM-ignoring pre-warm engine process is selected" + else + echo "FAIL: SIGTERM-ignoring pre-warm engine process was not selected" + failures=$((failures + 1)) + fi + + # 7. Containment: self_test issues REAL kills, and SELFTEST_SCOPE is the only + # thing keeping them inside the scratch tree. Assert that before killing + # rather than trusting it — this step runs on a hosted ephemeral lane + # today, but nothing in the script stops it being run anywhere else. + # The bystander decoy is what gives this check teeth. select_leaked filters + # every candidate against SELFTEST_SCOPE, and SELFTEST_SCOPE *is* ${tmp}, so + # the loop below — "is every selected entry inside ${tmp}?" — re-derives its + # answer from the very filter it claims to be checking, and cannot fail on + # any fixture that lives inside the scope. Deleting the filter outright left + # this check green. A rule-matching process OUTSIDE the scope is the only + # thing the filter can be caught NOT doing its job on. + # An empty selection satisfies the loop below trivially, so without this the + # "ok:" line would report containment verified on a run where nothing was + # examined. Checks 2 and 6 already fail such a run, so this is not a false + # green — but the evidence line is read by whoever is diagnosing that run. + # + # Counted apart from containment_failures on purpose: that counter arms the + # gate below, whose message and early return are specifically about a + # selection reaching OUTSIDE the scratch tree. An empty selection is the + # opposite failure and signals nothing at all, so routing it through that + # gate would report a false cause and cut the run short of checks 8-11. + # + # Diagnostic only, and said plainly rather than implied: an empty selection + # is already caught — checks 2 and 6 fail it — so deleting this check does + # NOT let such a run pass. What it adds is the reason, on the run someone + # is reading, which is also why the "ok:" line below is gated on it. + if [[ -z "${selected}" ]]; then + echo "FAIL: selection was empty; containment had nothing to examine" + failures=$((failures + 1)) + fi + outside_cmd="$(cmdline_of "${outside_pid}")" || outside_cmd='' + if [[ -z "${outside_cmd}" ]]; then + echo "FAIL: bystander decoy is not running; containment has no negative case" + containment_failures=$((containment_failures + 1)) + elif ! cmdline_matches_rule "${outside_cmd}"; then + # Asserted, not assumed: if the bystander stopped matching the rule it + # would be excluded for that reason instead of by the scope filter, and + # the negative case below would pass while proving nothing. + echo "FAIL: bystander decoy no longer matches the rule; containment proves nothing" + containment_failures=$((containment_failures + 1)) + elif grep -q "^${outside_pid} " <<<"${selected}"; then + echo "FAIL: a rule-matching process outside the scratch tree was selected" + containment_failures=$((containment_failures + 1)) + fi + while IFS=$'\t' read -r escapee_pid escapee_cmd; do + [[ -n "${escapee_pid}" ]] || continue + case "${escapee_cmd}" in + *"${tmp}"*) ;; + *) + echo "FAIL: selection escaped the scratch tree: pid=${escapee_pid} cmd=${escapee_cmd}" + containment_failures=$((containment_failures + 1)) + ;; + esac + done <<<"${selected}" + failures=$((failures + containment_failures)) + # Counted on its own rather than off the running total: gating this line on + # the total suppressed it whenever an EARLIER check had failed — which is + # precisely the run whose output someone is reading. + # Gated on a non-empty selection as well as on containment: the loop above is + # satisfied trivially by an empty one, so without this the run would print + # "selection was empty" and then immediately claim every selected process was + # checked — evidence contradicting itself on the run being diagnosed. + if [[ "${containment_failures}" -eq 0 && -n "${selected}" ]]; then + echo "ok: every selected process lies inside the self-test scratch tree" + elif [[ "${containment_failures}" -eq 0 ]]; then + : # empty selection: already reported above, and nothing to affirm here + else + # A GATE, not a score. Check 9 below calls the real `reclaim 0`, which + # sends real signals to whatever select_leaked returns at that moment. If + # containment has just failed, that selection reaches outside this scratch + # tree — the exact accident this check exists to prevent — so scoring it + # and carrying on would let the self-test do the damage it is guarding + # against. Stop here instead, while nothing has been signalled yet. + echo "FAIL: containment breached; refusing to run the real reclaim" + kill -KILL "${decoy_pids[@]}" 2>/dev/null || true + echo "reclaim-gpu self-test: ${failures} failure(s)" + return 1 + fi + + # 8. The escalation guard's comparison, in all three directions — and for + # "gone", by BOTH routes into it. + # + # The COMPARISON is covered here; check 9 covers the two CALL SITES by + # forcing the verdict, because a pid cannot be made to be reused by a + # different process on demand. + # The accept case needs the stubborn decoy alive to be comparable at all. If + # it has already exited, both reads fail and the guard answers "gone" — a + # dead fixture, not a guard that rejects a match, so it is reported as one. + guard_rc=0 + same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")" || guard_rc=$? + if [[ "${guard_rc}" == 0 ]]; then + echo "ok: escalation guard accepts an unchanged command line" + elif [[ "${guard_rc}" == 2 ]]; then + echo "FAIL: SIGTERM-ignoring decoy (pid=${stubborn_pid}) exited before the escalation guard could be checked" + failures=$((failures + 1)) + else + echo "FAIL: escalation guard rejected an unchanged command line; nothing would escalate" + failures=$((failures + 1)) + fi + guard_rc=0 + same_selected_process "${stubborn_pid}" "/some/other/process --unrelated" || guard_rc=$? + if [[ "${guard_rc}" == 1 ]]; then + echo "ok: escalation guard rejects a recycled pid" + else + echo "FAIL: escalation guard did not report a changed command line as recycled (rc=${guard_rc})" + failures=$((failures + 1)) + fi + guard_rc=0 + # A pid above /proc/sys/kernel/pid_max cannot exist, so this is the "gone" arm. + same_selected_process 2147483647 "anything" || guard_rc=$? + if [[ "${guard_rc}" == 2 ]]; then + echo "ok: escalation guard reports a departed process as gone, not recycled" + else + echo "FAIL: escalation guard conflated a departed process with a recycled one (rc=${guard_rc})" + failures=$((failures + 1)) + fi + # The other route into "gone": the open SUCCEEDS and reads nothing. The arm + # above exercises only a failed OPEN, so without this one `cmdline_of`'s + # emptiness check can be deleted with the self-test still green — restoring + # the "was recycled" mislabel for a process that merely exited, which is the + # race the guard exists to describe correctly. + if [[ "${zombie_pid}" != "none" ]]; then + guard_rc=0 + same_selected_process "${zombie_pid}" "anything" || guard_rc=$? + if [[ "${guard_rc}" == 2 ]]; then + echo "ok: escalation guard reports an argv-less zombie as gone, not recycled" + else + echo "FAIL: zombie with an empty command line was not reported as gone (rc=${guard_rc})" + failures=$((failures + 1)) + fi + else + # Failing rather than skipping: an arm that silently does not run is the + # exact defect this check was added to close. + echo "FAIL: could not produce a zombie decoy; the empty-cmdline arm went untested" + failures=$((failures + 1)) + fi + + # 9. The guard's two CALL SITES, by forcing the verdict they act on. + # + # A pid cannot be made to be reused by a different process on demand, so + # until now both call sites were exercised only in the always-proceed + # direction and either could be deleted with the self-test green. Overriding + # the comparison for one real `reclaim` run reaches them: with every verdict + # "recycled", a correct reclaim signals NOTHING, so deleting EITHER call + # site kills decoys here and fails the survival loop below. + # + # The override is declared INSIDE the command substitution, which bash runs + # in a subshell, so it cannot outlive this one call — no save/restore to get + # wrong, and no seam in the production path. The kills a broken guard would + # issue are still real, which is what makes the survival loop meaningful. + forced_out="$( + same_selected_process() { return 1; } + reclaim 0 + )" + sleep 1 + if grep -q "pid=${prewarm_pid} was recycled before it could be terminated" <<<"${forced_out}"; then + echo "ok: pre-TERM guard refused to signal a pid whose identity changed" + else + echo "FAIL: pre-TERM guard did not act on a recycled verdict" + failures=$((failures + 1)) + fi + # Bound to a specific pid, like its pre-TERM sibling above: an unbound match + # would be satisfied by this message emitted for any process. With every + # verdict forced to "recycled" nothing is signalled, so both engine decoys + # survive to the escalation loop and either pid would serve; the stubborn one + # is named because it is the decoy that reaches escalation in the UNforced + # run too, which keeps this assertion reading the same way as check 11. + if grep -q "pid=${stubborn_pid} was recycled during the grace period, not escalating" <<<"${forced_out}"; then + echo "ok: pre-KILL guard refused to escalate onto a recycled pid" + else + echo "FAIL: pre-KILL guard did not act on a recycled verdict" + failures=$((failures + 1)) + fi + # The point of both guards: a forced-recycled run must leave every process alive. + for guard_pid in "${decoy_pids[@]}"; do + if ! process_alive "${guard_pid}"; then + echo "FAIL: pid=${guard_pid} was signalled despite a recycled verdict" + failures=$((failures + 1)) + fi + done + # 10. End to end: reclaim kills the leaks and spares all three bystanders — + # the manual serve, the harness binary, and the out-of-scope decoy. + reclaim_out="$(reclaim 0)" + sleep 1 + if process_alive "${prewarm_pid}"; then + echo "FAIL: pre-warm engine process survived reclaim" + failures=$((failures + 1)) + else + echo "ok: pre-warm engine process was reclaimed" + fi + if process_alive "${workload_pid}"; then + echo "ok: /workload manual serve survived reclaim" + else + echo "FAIL: /workload manual serve was killed by reclaim" + failures=$((failures + 1)) + fi + if process_alive "${harness_pid}"; then + echo "ok: E2E test binary survived reclaim" + else + echo "FAIL: E2E test binary was killed by reclaim" + failures=$((failures + 1)) + fi + # The end-to-end form of check 3, and the one that states the cost: with the + # delimiters unflattened this process is selected, passed over as "recycled", + # and still holding the card when reclaim reports success. + if process_alive "${delimiter_pid}"; then + echo "FAIL: engine process with delimiters in its command line survived reclaim" + failures=$((failures + 1)) + else + echo "ok: engine process with delimiters in its command line was reclaimed" + fi + # The bystander matches the rule in full, so only the scope kept it out of the + # selection. Its survival is the end-to-end form of check 7. + if process_alive "${outside_pid}"; then + echo "ok: rule-matching process outside the scratch tree survived reclaim" + else + echo "FAIL: reclaim killed a rule-matching process outside the scratch tree" + failures=$((failures + 1)) + fi + + # 11. The escalation ran, and ran only where it was needed. Asserting the + # stubborn decoy died covers the SIGKILL block; asserting the ordinary + # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, + # which would otherwise be silently replaceable by any no-op. + if process_alive "${stubborn_pid}"; then + echo "FAIL: SIGTERM-ignoring process survived reclaim; escalation to SIGKILL did not happen" + failures=$((failures + 1)) + else + echo "ok: SIGTERM-ignoring process was escalated to SIGKILL" + fi + if grep -q "pid=${stubborn_pid} ignored SIGTERM" <<<"${reclaim_out}"; then + echo "ok: escalation was reported for the process that ignored SIGTERM" + else + echo "FAIL: no escalation reported for the SIGTERM-ignoring process" + failures=$((failures + 1)) + fi + if grep -q "pid=${prewarm_pid} ignored SIGTERM" <<<"${reclaim_out}"; then + echo "FAIL: ordinary decoy reached SIGKILL escalation; SIGTERM is not being delivered" + failures=$((failures + 1)) + else + echo "ok: ordinary decoy exited on SIGTERM without escalation" + fi + + # 12. report_holders' engine filter — the diagnostic every bash lane runs on a + # failed preflight. The bystanders survived check 10, so they are still + # there to be listed: the manual serve carries an engine marker and must + # appear, the harness binary carries none and must not. Deleting the + # marker derivation leaves grep with no pattern at all, which lists + # nothing and fails the first assertion. + holders_out="$(report_holders 2>&1)" + holders_engines="$(sed -n '/^--- engine\/serve processes/,/^--- of those/p' <<<"${holders_out}")" + if awk -v p="${workload_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "ok: report_holders lists a process carrying an engine marker" + else + echo "FAIL: report_holders did not list the manual serve (pid=${workload_pid})" + failures=$((failures + 1)) + fi + if awk -v p="${harness_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "FAIL: report_holders listed a process with no engine marker (pid=${harness_pid})" + failures=$((failures + 1)) + else + echo "ok: report_holders leaves out a process with no engine marker" + fi + # Fixed-string, not regex: under -E the `.` in this marker would match the + # `-` in the manual serve's path and list it. None of the real markers + # holds a metacharacter, so only an override can tell the two apart. + holders_out="$(ENGINE_MARKERS=('manual-serve/llama.server') && report_holders 2>&1)" + holders_engines="$(sed -n '/^--- engine\/serve processes/,/^--- of those/p' <<<"${holders_out}")" + if awk -v p="${workload_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "FAIL: report_holders matched a marker as a regex, not a literal substring" + failures=$((failures + 1)) + else + echo "ok: report_holders matches markers as literal substrings" + fi + + # 13. The bare invocation's rm -rf root. Asserted on the resolver rather than + # by running the deletion, which would clear the host's real scenario + # dirs. A relative TMPDIR must fall back to /tmp — the one branch whose + # failure is `rm -rf` against the CWD. + for tmp_case in 'relative/dir|/tmp' '|/tmp' '/abs/dir/|/abs/dir' '/abs/dir|/abs/dir'; do + if [[ "$(TMPDIR="${tmp_case%%|*}" scenario_tmp_root)" == "${tmp_case#*|}" ]]; then + echo "ok: TMPDIR='${tmp_case%%|*}' clears under ${tmp_case#*|}" + else + echo "FAIL: TMPDIR='${tmp_case%%|*}' did not resolve to ${tmp_case#*|}" + failures=$((failures + 1)) + fi + done + # The resolver is only half of it. Nothing above asserts that the `rm -rf` + # statement still CALLS it, so reverting that one line to an inline + # `${TMPDIR:-/tmp}` reintroduces the relative-TMPDIR hazard with the whole + # suite green — a covered helper nothing is obliged to use. Asserted + # against this file's own source, because actually running the deletion is + # what the paragraph above rules out. + if grep -qE '^[[:space:]]*rm -rf "\$\(scenario_tmp_root\)"' "${BASH_SOURCE[0]}"; then + echo "ok: the bare invocation clears its scenario dirs through scenario_tmp_root" + else + echo "FAIL: the rm -rf statement no longer calls scenario_tmp_root; TMPDIR handling is bypassed" + failures=$((failures + 1)) + fi + + # Includes prewarm_pid: on a GREEN run reclaim has already killed it, but on a + # FAILED run it was not selected, and it would otherwise outlive the scratch + # tree for its full 300s as an orphan. Killing the zombie's keeper lets init + # reap the zombie itself. + kill -KILL "${decoy_pids[@]}" 2>/dev/null || true + + if [[ "${failures}" -ne 0 ]]; then + echo "reclaim-gpu self-test: ${failures} failure(s)" + return 1 + fi + echo "reclaim-gpu self-test: all checks passed" +} + +# The directory the bare invocation clears scenario temp dirs under: TMPDIR when +# it is absolute, /tmp otherwise. Printed without a trailing slash. +# +# The absolute-path test is the SAME one the roots above apply, and it matters +# most here: this feeds the only consumer that deletes rather than merely +# failing to match. Interpolated raw, a relative TMPDIR makes the caller's +# `rm -rf relative/dir/rocm-e2e-*` resolve against the CWD — the repo checkout, +# on a CI runner. No lane sets one, which is a reason to skip the value, not a +# reason to hand it to `rm -rf` unchecked. A function rather than inline so the +# self-test can assert the fallback without deleting anything. +scenario_tmp_root() { + local root="${TMPDIR:-}" + [[ "${root}" == /* ]] || root='/tmp' + printf '%s' "${root%/}" +} + +main() { + case "${1:-}" in + '') + reclaim 0 + # Scenario temp dirs are recreated per run; clearing them keeps a wedged + # runner's disk from filling with dead scenario state. + # + # TMPDIR as well as /tmp, because the suite creates these through + # `std::env::temp_dir()` and the nightly Strix lane redirects TMPDIR to + # the runner's home. A bare `/tmp` glob removed nothing at all there, so + # the guard was dead on the one lane that redirects. Both are listed + # rather than just TMPDIR: the lanes that do not set it still want /tmp, + # and a stale tree from before a redirect was added would outlive it. + rm -rf "$(scenario_tmp_root)"/rocm-e2e-* /tmp/rocm-e2e-* 2>/dev/null || true + ;; + --dry-run) reclaim 1 ;; + --report-holders) report_holders ;; + --self-test) self_test ;; + -h | --help) usage ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +} + +main "$@" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 13506fee31..b757d4ae55 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2072,4 +2072,392 @@ permissions: let block = " env:\n VALID: one\n MALFORMED\n steps:\n"; let _ = job_mapping(block, "env"); } + + /// The E2E-owned roots declared in `scripts/reclaim-gpu.sh`. + /// + /// Parsed rather than duplicated: a copy here would drift the same way the + /// PowerShell mirrors can, which is the defect this test exists to prevent. + fn reclaim_script_roots() -> Vec { + let p = repo_root().join("scripts/reclaim-gpu.sh"); + let text = std::fs::read_to_string(&p) + .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + .replace("\r\n", "\n"); + parse_e2e_roots(&text, &p.display().to_string()) + } + + /// The parsing half of [`reclaim_script_roots`], split out so every spelling + /// it must cope with is a standing test rather than a mutation somebody has + /// to remember to run by hand against the tracked script. + /// + /// That split is the point. The previous version of this parser could only + /// be exercised by editing `scripts/reclaim-gpu.sh` itself, so three legal + /// spellings of the same array reached review unnoticed — and the failure + /// they produced blamed the PowerShell mirrors, advising a maintainer to + /// delete a real root from Windows. + fn parse_e2e_roots(text: &str, origin: &str) -> Vec { + // Comments come off BEFORE anything is located — before the array's + // START as well as its closing `)`. Two distinct mis-parses, one fix: + // + // - an inline comment containing a `)` — `'/tmp/rocm-e2e' # see docs + // (section 3)` — closes the array at that paren, so every root below + // it vanishes from the parse. The test then fails in the + // mirror->script direction naming a root that is plainly still in + // the script, and a maintainer who follows that message deletes a + // real root from both PowerShell mirrors. Measured: that spelling + // made this test demand the deletion of `e2e-prewarm`, the root this + // whole change exists to add. + // - a comment ABOVE the array that merely quotes the literal text + // `E2E_ROOTS=(` would otherwise anchor the search inside that + // sentence rather than at the declaration. This file already quotes + // bash array syntax in prose (`E2E_ROOTS+=(` appears in a comment), + // so that is a spelling a future header edit can reach. + let decommented = text + .lines() + .map(|l| { + if l.trim_start().starts_with('#') { + "" + } else { + strip_comment(l) + } + }) + .collect::>() + .join("\n"); + let body = decommented + .split_once("E2E_ROOTS=(") + .unwrap_or_else(|| panic!("{origin} must declare E2E_ROOTS=(")) + .1 + .split_once(')') + .unwrap_or_else(|| panic!("{origin} has an unterminated E2E_ROOTS array")) + .0; + + // Every non-empty line inside the array must parse into exactly one + // root. This is the load-bearing half: without it an unrecognised + // spelling is silently DROPPED, and a dropped root is indistinguishable + // from mirror drift downstream — so the failure arrives as a confident + // wrong remedy rather than as "this parser did not understand the + // array". A checker that converts a formatting change into advice that + // removes Windows coverage is worse than no checker on that axis. + // + // Deliberately NOT tokenised on whitespace: that would silently split a + // future root containing a space into two bogus roots, which is this + // same defect class in a new place. + let mut roots: Vec = Vec::new(); + for line in body.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + // Both bash quotings are accepted. A quoted root can never contain + // its OWN delimiter, so requiring that is what rejects the whole + // array folded onto one line — `E2E_ROOTS=('a' 'b')` parses as the + // single bogus root `a' 'b` under a prefix/suffix strip alone, and + // then fails script->mirror naming a root nobody wrote. + let parsed = ['\'', '"'].into_iter().find_map(|q| { + line.strip_prefix(q) + .and_then(|l| l.strip_suffix(q)) + .filter(|inner| !inner.contains(q)) + }); + let Some(root) = parsed else { + // `strip_comment` splits on a literal " #" with no quote + // awareness, so a root whose VALUE contains that sequence — + // `'/tmp/e2e root #2'`, which bash itself accepts, since `#` is + // inert inside single quotes — arrives here already truncated + // and unterminated. Named in the message rather than fixed: + // this lands as a loud false failure, never a silent miss, and + // making the shared helper quote-aware for a spelling no root + // uses would be untested code on a path every other caller + // depends on. + panic!( + "{origin}: E2E_ROOTS line `{line}` is not a single quoted root. This \ + test's parser did not understand it — that is NOT drift against the \ + PowerShell mirrors, so do not remove anything from them. Restore one \ + quoted root per line, or teach this parser the new spelling. Note the \ + line is shown after comment-stripping, so a root whose value contains \ + \" #\" appears truncated here (EAI-8751)" + ); + }; + roots.push(root.to_owned()); + } + assert!( + !roots.is_empty(), + "{origin} declared no E2E_ROOTS entries — the parser or the array shape changed" + ); + roots + } + + /// An `E2E_ROOTS` array spelled the way the script spells it today, with + /// enough surrounding file to exercise the comment handling. + #[cfg(test)] + fn roots_fixture(body: &str) -> String { + format!( + "#!/usr/bin/env bash\n# a header comment\nset -euo pipefail\nE2E_ROOTS=(\n{body})\nENGINE_MARKERS=(\n 'llama-server'\n)\n" + ) + } + + /// Every spelling this parser must cope with, as a test rather than as a + /// mutation of the tracked script. Each of these was found by editing + /// `scripts/reclaim-gpu.sh` by hand during review; none of them could fail + /// CI afterwards, which is the gap this closes. + #[test] + fn e2e_roots_parser_accepts_legal_respellings_of_the_same_array() { + let canonical = vec!["/tmp/rocm-e2e".to_owned(), "e2e-shared".to_owned()]; + + // As written today. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e'\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // An inline comment containing a `)`. This closed the array early and + // made the test demand the deletion of a real root from Windows. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e' # see docs (section 3)\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // Double quotes, and a mix of both. Silently dropped the entry before. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" \"/tmp/rocm-e2e\"\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // A full-line comment inside the array, indented or at column 0, and a + // blank line. All three are skipped rather than failing completeness. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e'\n# flush left\n # indented\n\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // A comment ABOVE the array quoting the declaration's own text. The + // anchor must find the declaration, not the sentence about it. + let text = "#!/usr/bin/env bash\n# see how E2E_ROOTS=( is declared below\nE2E_ROOTS=(\n '/tmp/rocm-e2e'\n 'e2e-shared'\n)\n"; + assert_eq!(parse_e2e_roots(text, "fixture"), canonical); + + // A root containing a space is preserved whole, which is what rules out + // tokenising the body on whitespace. + assert_eq!( + parse_e2e_roots(&roots_fixture(" '/tmp/rocm e2e'\n"), "fixture"), + vec!["/tmp/rocm e2e".to_owned()] + ); + } + + /// The other half: spellings the parser must REJECT, and reject as its own + /// misunderstanding rather than as drift against the PowerShell mirrors — + /// because the mirror-drift message tells a maintainer to delete a root + /// from Windows, and following it on a false positive is what makes this + /// worse than having no checker. + #[test] + fn e2e_roots_parser_rejects_unparsable_lines_without_blaming_the_mirrors() { + for (label, body) in [ + ( + "array folded onto one line", + " '/tmp/rocm-e2e' 'e2e-shared'\n", + ), + ("unquoted entry", " e2e-shared\n"), + ("unterminated quote", " '/tmp/rocm-e2e\n"), + // `strip_comment` is not quote-aware, so this arrives truncated. + // Rejected loudly, never silently dropped. + ("value containing \" #\"", " '/tmp/rocm e2e #2'\n"), + ] { + let text = roots_fixture(body); + let err = std::panic::catch_unwind(|| parse_e2e_roots(&text, "fixture")) + .expect_err(&format!("{label} must not parse")); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("did not understand"), + "{label} failed with the wrong message — it must not read as mirror drift: {msg}" + ); + assert!( + msg.contains("do not remove anything from them"), + "{label} must tell the reader NOT to edit the mirrors: {msg}" + ); + } + } + + /// An empty array is a parser/shape change, not zero roots — and must not + /// sail through as "no roots declared, nothing to compare". + #[test] + fn e2e_roots_parser_rejects_an_empty_array() { + let err = std::panic::catch_unwind(|| parse_e2e_roots(&roots_fixture(""), "fixture")) + .expect_err("an empty E2E_ROOTS must not parse as success"); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!(msg.contains("declared no E2E_ROOTS entries"), "got: {msg}"); + } + + /// The root half of each PowerShell reclaim: the FIRST `-match '…'` + /// alternation on the `Where-Object` line. + /// + /// This cannot tell the reclaim's own matcher from any other + /// `Where-Object … -match` line. Each of these workflows has exactly one + /// today, so every alternation returned IS a reclaim matcher; add a second, + /// unrelated one and the caller's assertions would be applied to it too. + /// + /// Extracted rather than substring-matched against the whole file for the + /// reason this module's header gives: every root ALSO appears in these + /// workflows as an env var and in prose, so a `text.contains(root)` check + /// passes even when the alternation itself has lost that root. Confirmed by + /// mutation — deleting `e2e-prewarm` from the alternation left the + /// whole-file form of this test green. + fn powershell_reclaim_root_alternations(text: &str) -> Vec { + text.lines() + .filter(|l| l.contains("Where-Object") && l.contains("-match")) + .map(|l| { + let after = l.split_once("-match").expect("filtered on -match").1; + let body = after + .split_once('\'') + .unwrap_or_else(|| panic!("no opening quote in matcher line: {l}")) + .1; + body.split_once('\'') + .unwrap_or_else(|| panic!("unterminated matcher literal: {l}")) + .0 + .to_owned() + }) + .collect() + } + + /// EAI-8751: native Windows has no bash, so the two PowerShell reclaim steps + /// restate the bash rule instead of sharing it. Nothing else in CI compares + /// the two, so a root added to the script — as `e2e-prewarm` was, to fix a + /// leak that held the card for 16 consecutive jobs — can silently miss the + /// Windows lanes. + /// + /// Only the ROOT half is pinned. The engine half is knowingly divergent on + /// Windows (EAI-8815), so asserting parity there would fail on a difference + /// that is recorded rather than accidental. + #[test] + fn reclaim_roots_are_mirrored_in_the_powershell_reclaims() { + let roots = reclaim_script_roots(); + for workflow in ["e2e-selfhosted.yml", "nightly.yml"] { + let text = read_workflow(workflow); + assert!( + text.contains("Get-CimInstance Win32_Process"), + "{workflow} must keep a PowerShell reclaim step (EAI-8751)" + ); + let alternations = powershell_reclaim_root_alternations(&text); + assert!( + !alternations.is_empty(), + "{workflow} has a PowerShell reclaim but no parsable `-match` alternation \ + — the step's shape changed and this guard went blind (EAI-8751)" + ); + // The scenario root is compared on its portable segment, and that + // asymmetry is deliberate rather than cosmetic. + // + // bash anchors it to an absolute path (`/tmp/rocm-e2e`, plus a + // TMPDIR-derived form appended at run time) because its roots are + // matched as unanchored substrings of a whole command line: the + // bare segment would also match an ARGUMENT naming it, and kill a + // hand-run serve the script promises to spare. The PowerShell + // mirrors carry the bare segment and so do have that exposure — + // pre-existing, and not something this test can fix by failing. + // + // That anchoring gap is NOT what EAI-8815 tracks. That ticket is + // scoped to the engine-marker divergence (`rocm.exe daemon` + // unmatched; `__engine-serve-http` sitting in the root alternation + // rather than the engine one), and closing it would leave the + // substring exposure untouched. The anchoring gap is untracked on + // the Windows side — said plainly here, so that closing EAI-8815 + // cannot be misread as closing this as well. + // + // What it still pins is the part that matters here: that every root + // the script knows about is named in both mirrors, so a root added + // to one side cannot silently miss the Windows lanes. + // + // CAVEAT: the `/tmp/` prefix below is the only absolute prefix this + // mapping knows. A future root anchored under any other one — say + // `/var/tmp/rocm-e2e` — would be compared to the mirrors verbatim, + // fail against their bare segment, and need this line edited by + // hand: the manual sync this test exists to remove, reappearing one + // level up. Named rather than generalised, because there is exactly + // one anchored root today and a speculative prefix list would be + // untested code. + let expected: Vec<&str> = roots + .iter() + .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) + .collect(); + + // Pinned at one, which is what lets the exemption below be tracked + // per WORKFLOW rather than per alternation. Requiring EVERY + // alternation to name the exempted marker would fail spuriously the + // moment an unrelated `Where-Object … -match` line appeared — the + // case `powershell_reclaim_root_alternations` warns it cannot + // distinguish — but relaxing it to "some alternation" would stop + // catching a second matcher that copies every root and drops the + // marker. While there is exactly one, the two readings coincide; + // this assertion is what keeps that true, and fails loudly with + // something to decide if a second one is ever added. + assert_eq!( + alternations.len(), + 1, + "{workflow} now has {} PowerShell reclaim matcher alternations; the \ + divergence-exemption check below assumes exactly one, and must be \ + re-read per alternation before this count changes (EAI-8751)", + alternations.len() + ); + let mut divergence_seen = false; + for alternation in &alternations { + let present: Vec<&str> = alternation.split('|').collect(); + + // Script -> mirror: a root added to the script must reach Windows. + for needle in &expected { + assert!( + present.contains(needle), + "{workflow}'s PowerShell reclaim matcher `{alternation}` does not name \ + the E2E root `{needle}` declared in scripts/reclaim-gpu.sh (EAI-8751)" + ); + } + + // Mirror -> script: and a root REMOVED from the script must not be + // left behind here. Without this direction the guard is one-way, + // which is how the lists drifted in the first place. + for token in &present { + // Known divergence, not drift: `__engine-serve-http` is an + // ENGINE marker that sits in this root alternation, so on + // Windows it over-matches. Tracked in EAI-8815 — when that is + // fixed, delete this arm and the assertion below will hold. + if *token == "__engine-serve-http" { + divergence_seen = true; + continue; + } + assert!( + expected.contains(token), + "{workflow}'s PowerShell reclaim matcher `{alternation}` names `{token}`, \ + which is not an E2E root in scripts/reclaim-gpu.sh — remove it here too, \ + or add it there (EAI-8751)" + ); + } + } + + // An exemption nothing asserts is an exemption that rots: once + // EAI-8815 moves `__engine-serve-http` out of the root alternation, + // the arm above stops firing and would sit here forever as dead + // code exempting nothing. Fail instead, so the fix is told to + // finish the job. + assert!( + divergence_seen, + "no PowerShell reclaim matcher in {workflow} names `__engine-serve-http`, so the \ + EAI-8815 divergence looks fixed — delete the exemption arm in this test, which \ + is now dead code (EAI-8751)" + ); + } + } }