From 58a93aa8d85034eb0bccc083428b21bcc8753a39 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 15 Sep 2026 11:22:38 +0000 Subject: [PATCH 01/16] ci(e2e): reclaim engine processes leaked into the pre-warm tree (EAI-8751) The GPU reclaim step matched a leaked llama-server two ways only: anchored to a scenario temp dir (/tmp/rocm-e2e) or to the shared cache (e2e-shared). A managed serve runs the engine binary out of the shared pre-warm runtime tree instead, so neither pattern could see it. Both were also order-sensitive: pkill -f matches the whole command line, and argv[0] is the binary path, so a /tmp/rocm-e2e model argument later in the line could not rescue them either. A leak therefore survived every later job's reclaim and held the card until the runner itself was replaced. On the container lanes the pod is fresh per job but the work volume is not, so one leak failed every job scheduled to that runner at the GPU preflight -- 16 consecutive jobs over 2h20m on the R9700 lane, all reporting the same 8 GiB free. Replace eleven copies of the inline pkill list with scripts/reclaim-gpu.sh, which requires an E2E-owned root AND an engine marker in any order, covers the pre-warm tree, escalates TERM to KILL, and reports what it killed rather than printing an unconditional "reclaimed". Native Windows has no bash, so its two steps mirror the same rule in PowerShell. --self-test proves the rule against decoy processes with no GPU present: it asserts the superseded patterns miss a pre-warm engine process, that the new rule selects and kills it, and that a manual /workload serve on a shared runner is left alone. Wired into the existing self-test block on the GitHub-hosted CI lane. On preflight failure the lanes now call --report-holders, which prints per-process VRAM and the engine processes present; reconstructing that by hand is what made this slow to diagnose. Signed-off-by: Tomas Saaristola --- .github/workflows/ci.yml | 7 + .github/workflows/e2e-selfhosted.yml | 107 ++++----- .github/workflows/nightly.yml | 89 ++++---- scripts/reclaim-gpu.sh | 314 +++++++++++++++++++++++++++ 4 files changed, 429 insertions(+), 88 deletions(-) create mode 100755 scripts/reclaim-gpu.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65f570109c..4d22b24e63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,6 +377,13 @@ jobs: if: needs.changes.outputs.heavy == 'true' run: bash scripts/setup-wsl-portable-build-deps.sh --self-test + # Runs here, on the GitHub-hosted lane, rather than on the GPU runners the + # reclaim protects: the matching rules are what regress, and they are + # provable against decoy processes with no GPU present. + - name: GPU reclaim self-test + if: needs.changes.outputs.heavy == 'true' + run: bash scripts/reclaim-gpu.sh --self-test + - name: Release readiness self-test if: needs.changes.outputs.heavy == 'true' run: python scripts/release_readiness.py --self-test diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index b44bf797e1..926cb7b3c8 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -177,25 +177,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Reclaim the GPU before running: a serve leaked by a killed/timed-out prior - # run (its Drop teardown never executed) can keep an engine process spinning - # on the GPU, starving this job's serves until it hits the timeout. Kill only - # e2e leftovers — scoped to /tmp/rocm-e2e-* and the e2e-target/e2e-shared - # trees — never the runner or any /workload manual-testing processes. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - # NOTE: no unanchored `pkill -f 'vulkan/llama-server'` — the lemonade - # Vulkan assistant an e2e scenario spawns lives under /tmp/rocm-e2e-* - # and is already caught by the first line; an unanchored pattern would - # also kill a legitimate /workload manual-testing serve on this shared - # self-hosted runner. - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the # GPU is missing, the driver is wedged, or VRAM is still saturated by a @@ -235,6 +223,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false — on this self-hosted runner we persist the build cache @@ -409,14 +401,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: bounded wait so a missing/wedged/held GPU fails fast (~90s) # instead of hanging to the job cap. Transient contention (VRAM still @@ -557,6 +548,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # Pool hosts don't preinstall cargo/rustup; bootstrap ourselves with @@ -673,10 +668,18 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaimed" + # Same rule as scripts/reclaim-gpu.sh, which is the source of truth for + # these markers: an E2E-owned root AND an engine process. Native Windows + # has no bash, so the rule is mirrored here rather than shared. Keep the + # root list in step with the script — `e2e-prewarm` is the shared runtime + # tree the engine binaries actually run from. + $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } + foreach ($p in $leaked) { + Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" + Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue + } + Write-Host "reclaim: $(@($leaked).Count) process(es) terminated" # GPU preflight (best-effort on Windows): bounded wait for the GPU to be # available so a wedged/held GPU fails fast instead of hanging to the cap. @@ -1211,6 +1214,10 @@ jobs: exit 0 fi echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${pool} never dropped below the floor — a serve is likely still holding the GPU" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does @@ -1378,18 +1385,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pod is usually fresh (scale-to-zero recreates it per job), but the - # work PVC is not, and two jobs can share one pod's life. Same reclaim as - # e2e-gpu, scoped to e2e leftovers only. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait, as in e2e-gpu. The pod requests `amd.com/gpu: 1`, so # rocm-smi sees exactly the one card Kueue admitted it for. The floor is @@ -1420,6 +1422,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false for the same reason as e2e-gpu — the build cache is kept in @@ -1506,18 +1512,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pod is usually fresh (the runner scales to zero between jobs), but the - # work volume is not, and two jobs can share one pod's life. Same reclaim as - # e2e-gpu, scoped to e2e leftovers only. + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait, as in e2e-gpu. The pod is admitted for exactly one card, so # rocm-smi sees that card alone. The floor matches the other Instinct lane @@ -1549,6 +1550,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false for the same reason as e2e-gpu — the build cache is kept in diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 67e6181147..31a38e6ef5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -370,18 +370,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Reclaim the GPU before running: kill only e2e leftovers scoped to - # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or - # any /workload manual-testing processes (see e2e-selfhosted.yml e2e-gpu for the same). + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the # GPU is missing, the driver is wedged, or VRAM is still saturated by a @@ -421,6 +416,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # cache: false — this self-hosted runner persists the build cache itself via @@ -494,15 +493,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU. Floor is half of the R9700S's 32GB — high # enough to catch a leftover serve, low enough that a clean pod passes on @@ -532,6 +529,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 @@ -585,15 +586,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Reclaim the GPU before running: a serve leaked by a killed or + # timed-out prior run (its Drop teardown never executed) keeps an engine + # process on the card and starves this job's serves. Scoped to E2E-owned + # trees, so a manual /workload serve on a shared self-hosted runner is + # never touched. See scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes - run: | - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "reclaimed" + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU. Floor matches the per-PR MI350P lane and the # other Instinct lane — high enough to catch a leftover serve, low enough @@ -623,6 +622,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 @@ -694,12 +697,8 @@ jobs: - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs run: | mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" - pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true - pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true - pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true - pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true - echo "prepared + reclaimed" + echo "prepared" + bash scripts/reclaim-gpu.sh - name: GPU preflight (bounded wait for an available GPU) run: | @@ -831,6 +830,10 @@ jobs: sleep 5 done echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 - name: Ensure Rust toolchain @@ -887,10 +890,18 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | - Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | - ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaimed" + # Same rule as scripts/reclaim-gpu.sh, which is the source of truth for + # these markers: an E2E-owned root AND an engine process. Native Windows + # has no bash, so the rule is mirrored here rather than shared. Keep the + # root list in step with the script — `e2e-prewarm` is the shared runtime + # tree the engine binaries actually run from. + $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } + foreach ($p in $leaked) { + Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" + Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue + } + Write-Host "reclaim: $(@($leaked).Count) process(es) terminated" # Best-effort when rocm-smi is absent; when present, fail within the bounded # ceiling instead of letting a held or wedged GPU consume the full job cap. @@ -1329,6 +1340,10 @@ jobs: exit 0 fi echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${pool} never dropped below the floor — a serve is likely still holding the GPU" + # Name what is holding the card instead of leaving the next reader to + # reconstruct it: --report-holders prints per-process VRAM and the + # engine processes visible here. Never fails the step itself. + bash scripts/reclaim-gpu.sh --report-holders || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh new file mode 100755 index 0000000000..71d072b0c2 --- /dev/null +++ b/scripts/reclaim-gpu.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +# Reclaim the GPU from E2E processes leaked by a killed or timed-out prior run, +# whose Drop teardown never executed. +# +# WHY A SCRIPT AND NOT INLINE `pkill -f`: the inline version anchored each +# pattern to a scenario temp dir (`/tmp/rocm-e2e`) or the shared *cache* +# (`e2e-shared`), but a managed serve runs the engine binary out of the shared +# *pre-warm runtime* tree — `$RUNNER_WORKSPACE/e2e-prewarm-*/data/runtimes/...` +# — which matched neither. `pkill -f` also matches the whole command line in +# order, and argv[0] is the binary path, so a `/tmp/rocm-e2e-*` model argument +# appearing later could not rescue those patterns either. A leaked llama-server +# was therefore invisible to reclaim and held the card until the runner itself +# was replaced, failing every later job on that runner at the GPU preflight. +# +# That is why a miss here is expensive rather than merely untidy: on the +# container lanes the pod is usually fresh per job, but the work volume is not +# and two jobs can share one pod's life — so a single leak the reclaim cannot +# match poisons every job scheduled to that runner until the pod is recreated. +# +# The rule here is order-independent and covers every E2E-owned tree: a process +# is reclaimed when its command line names an E2E root AND an engine/serve +# process. Both halves are required, which is what keeps a legitimate +# `/workload` manual-testing serve on a shared self-hosted runner safe — it +# names no E2E root. + +set -euo pipefail + +# E2E-owned roots. A process must name one of these to be considered ours. +# /tmp/rocm-e2e per-scenario temp dirs +# e2e-shared E2E_SHARED_CACHE_DIR (models, HF weights) +# e2e-prewarm E2E_SHARED_RUNTIMES_DIR — the pre-warmed runtime tree the +# engine binaries actually live in +# e2e-target CARGO_TARGET_DIR for the suite's own binaries +E2E_ROOTS=( + '/tmp/rocm-e2e' + 'e2e-shared' + 'e2e-prewarm' + 'e2e-target' +) + +# Engine/serve processes that can hold VRAM. Matched anywhere in the command +# line, so a wrapper or an absolute binary path both work. +ENGINE_MARKERS=( + 'llama-server' + 'vllm' + '__engine-serve-http' + 'rocm daemon' +) + +# Seconds to wait for a TERM'd process to exit before escalating to KILL. +TERM_GRACE_SECS="${RECLAIM_TERM_GRACE_SECS:-5}" + +# Set by --self-test so its decoys are the only things in scope. Never set in +# CI: an empty scope means "any process matching the rules above". +SELFTEST_SCOPE="${RECLAIM_SELFTEST_SCOPE:-}" + +usage() { + cat <<'EOF' +Usage: reclaim-gpu.sh [--dry-run | --report-holders | --self-test] + + (no flags) Terminate leaked E2E engine processes and report what was killed. + --dry-run List what would be terminated; kill nothing. + --report-holders Print current GPU holders and candidate processes, for + diagnosing a preflight failure. Kills nothing, never fails. + --self-test Verify the matching rules against decoy processes. No GPU needed. +EOF +} + +# True while a pid is a live process. A zombie is NOT alive: it has already +# released its VRAM and is only waiting to be reaped, but `kill -0` still +# succeeds on it, which would read as "ignored SIGTERM". +process_alive() { + local pid="$1" + local state + # /proc//stat field 3 is the state, but field 2 (comm) may contain + # spaces, so cut after the closing paren rather than counting fields. + state="$(sed 's/.*) //' "/proc/${pid}/stat" 2>/dev/null | cut -d' ' -f1)" || return 1 + [[ -n "${state}" ]] || return 1 + [[ "${state}" != "Z" ]] +} + +# Print "pidcommand line" for every process whose command line names both +# an E2E root and an engine marker. +select_leaked() { + local cmdline_file pid cmdline root engine has_root has_engine + for cmdline_file in /proc/[0-9]*/cmdline; do + pid="${cmdline_file#/proc/}" + pid="${pid%/cmdline}" + # Never reclaim ourselves or our own shell. + if [[ "${pid}" == "$$" || "${pid}" == "${PPID}" ]]; then + continue + fi + # A process can exit between the glob and the read; that is not an error. + cmdline="$(tr '\0' ' ' <"${cmdline_file}" 2>/dev/null)" || continue + [[ -n "${cmdline}" ]] || continue + + if [[ -n "${SELFTEST_SCOPE}" ]]; then + case "${cmdline}" in + *"${SELFTEST_SCOPE}"*) ;; + *) continue ;; + esac + fi + + has_root=0 + for root in "${E2E_ROOTS[@]}"; do + case "${cmdline}" in + *"${root}"*) + has_root=1 + break + ;; + *) ;; + esac + done + [[ "${has_root}" == 1 ]] || continue + + has_engine=0 + for engine in "${ENGINE_MARKERS[@]}"; do + case "${cmdline}" in + *"${engine}"*) + has_engine=1 + break + ;; + *) ;; + esac + done + [[ "${has_engine}" == 1 ]] || continue + + printf '%s\t%s\n' "${pid}" "${cmdline}" + done +} + +# TERM, wait out the grace period, then KILL whatever is left. Reports every +# process it acts on: an unconditional "reclaimed" tells the next reader +# nothing, and a silent no-op is how the original defect stayed hidden. +reclaim() { + local dry_run="$1" + local selected pid cmdline + local killed=0 + local waited=0 + + selected="$(select_leaked)" + if [[ -z "${selected}" ]]; then + echo "reclaim: no leaked E2E engine processes found" + return 0 + fi + + while IFS=$'\t' read -r pid cmdline; do + [[ -n "${pid}" ]] || continue + if [[ "${dry_run}" == 1 ]]; then + echo "reclaim: would terminate pid=${pid} cmd=${cmdline}" + else + echo "reclaim: terminating pid=${pid} cmd=${cmdline}" + kill -TERM "${pid}" 2>/dev/null || true + fi + killed=$((killed + 1)) + done <<<"${selected}" + + if [[ "${dry_run}" == 1 ]]; then + echo "reclaim: ${killed} process(es) would be terminated (dry run)" + return 0 + fi + + while [[ "${waited}" -lt "${TERM_GRACE_SECS}" ]]; do + if [[ -z "$(select_leaked)" ]]; then + break + fi + sleep 1 + waited=$((waited + 1)) + done + + # Anything still alive after the grace period gets SIGKILL. VRAM is released + # by the kernel when the process dies, so this is what actually frees the card. + while IFS=$'\t' read -r pid cmdline; do + [[ -n "${pid}" ]] || continue + if process_alive "${pid}"; then + echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" + kill -KILL "${pid}" 2>/dev/null || true + fi + done <<<"${selected}" + + echo "reclaim: ${killed} process(es) terminated" +} + +# Diagnostics for a preflight that hit its ceiling. Never fails: it runs on the +# failure path, where masking the real error would be worse than missing output. +report_holders() { + echo "--- rocm-smi KFD processes (per-process VRAM) ---" + # Process names show as UNKNOWN inside a container: KFD reports host PIDs, + # which do not resolve in the container's PID namespace. The VRAM column is + # still the answer to "what is holding the card". + timeout 15 rocm-smi --showpids 2>&1 | head -40 || true + echo "--- engine/serve processes visible here ---" + # shellcheck disable=SC2009 # pgrep cannot print elapsed time, and how long a + # holder has been alive is what distinguishes a leak from this job's own serve. + ps -eo pid,etimes,args 2>/dev/null | + grep -Ei 'llama-server|vllm|__engine-serve-http|rocm daemon' | + grep -v grep | + head -40 || true + echo "--- of those, E2E-owned (reclaim would take these) ---" + select_leaked || true +} + +# Spawn a decoy whose argv[0] is a real-shaped path, so the match is tested +# against the same string a live engine process would present. +spawn_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + cp /bin/sleep "${path}" + # Detach stdio: this runs inside a command substitution, and a background + # child holding the capture pipe open would block the caller until it exits. + "${path}" 300 >/dev/null 2>&1 & + echo $! +} + +self_test() { + local tmp prewarm_decoy workload_decoy prewarm_pid workload_pid selected + local failures=0 + + # Deliberately NOT under /tmp/rocm-e2e: that prefix is one of the roots the + # old patterns did match, which would mask the regression this guards. + tmp="$(mktemp -d /tmp/reclaim-selftest-XXXXXX)" + export RECLAIM_SELFTEST_SCOPE="${tmp}" + SELFTEST_SCOPE="${tmp}" + # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time + trap "rm -rf '${tmp}'" EXIT + + # The real shape: lemonade's engine binary inside the shared pre-warm runtime. + prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" + # A manual-testing serve on a shared runner: an engine, but no E2E root. + workload_decoy="${tmp}/workload/manual-serve/llama-server" + + prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" + workload_pid="$(spawn_decoy "${workload_decoy}")" + # Give both decoys a moment to appear in /proc with their full argv. + sleep 1 + + # 1. Regression guard: the patterns this script replaced could not see a + # pre-warm engine process. If this ever matches, the decoy stopped being + # representative and the rest of the self-test proves nothing. + if pgrep -f '/tmp/rocm-e2e.*llama-server' >/dev/null 2>&1 || + pgrep -f 'e2e-shared.*llama-server' >/dev/null 2>&1; then + echo "FAIL: superseded patterns matched the pre-warm decoy; decoy is unrepresentative" + failures=$((failures + 1)) + else + echo "ok: superseded patterns do not match a pre-warm engine process (the defect)" + fi + + # 2. The new rule selects it. + selected="$(select_leaked)" + if grep -q "^${prewarm_pid} " <<<"${selected}"; then + echo "ok: pre-warm engine process is selected" + else + echo "FAIL: pre-warm engine process was not selected" + failures=$((failures + 1)) + fi + + # 3. A manual-testing serve is left alone. + if grep -q "^${workload_pid} " <<<"${selected}"; then + echo "FAIL: /workload manual serve was selected; reclaim must not touch it" + failures=$((failures + 1)) + else + echo "ok: /workload manual serve is not selected" + fi + + # 4. End to end: reclaim actually kills the leak and spares the bystander. + reclaim 0 >/dev/null + sleep 1 + if process_alive "${prewarm_pid}"; then + echo "FAIL: pre-warm engine process survived reclaim" + failures=$((failures + 1)) + else + echo "ok: pre-warm engine process was reclaimed" + fi + if process_alive "${workload_pid}"; then + echo "ok: /workload manual serve survived reclaim" + else + echo "FAIL: /workload manual serve was killed by reclaim" + failures=$((failures + 1)) + fi + + kill -KILL "${workload_pid}" 2>/dev/null || true + + if [[ "${failures}" -ne 0 ]]; then + echo "reclaim-gpu self-test: ${failures} failure(s)" + return 1 + fi + echo "reclaim-gpu self-test: all checks passed" +} + +main() { + case "${1:-}" in + '') + reclaim 0 + # Scenario temp dirs are recreated per run; clearing them keeps a wedged + # runner's disk from filling with dead scenario state. + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + ;; + --dry-run) reclaim 1 ;; + --report-holders) report_holders ;; + --self-test) self_test ;; + -h | --help) usage ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +} + +main "$@" From cf6902a67e160e8a269fe21e1bb7a1766d248ef0 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 17 Sep 2026 14:10:36 +0000 Subject: [PATCH 02/16] ci(e2e): fail the reclaim self-test on the branches it guards (EAI-8751) The self-test passed with either half of the new matching rule removed, so it gated only a wholesale revert of the roots list: - every decoy carried an engine marker, and the sole bystander was excluded by the root half alone, so deleting `has_engine` left it green; - both decoys were copies of /bin/sleep, which exit on SIGTERM, so the TERM -> grace -> KILL escalation was never reached and either of its two branches could be removed with the test still green. Add the two decoys that close this: an E2E-owned path carrying no engine marker (the suite's own binary under CARGO_TARGET_DIR, which is exactly what `has_engine` spares) and a SIGTERM-ignoring engine at a pre-warm-shaped path. Assert escalation was reported for the stubborn decoy and NOT for the ordinary one, which is what pins the `kill -TERM` preceding it. Verified by mutation: gutting `has_engine`, deleting the SIGKILL block, and replacing `kill -TERM` with a no-op each fail the self-test now; all three passed it before. Also: - do not escalate to SIGKILL on a pid whose command line changed during the grace window, so a recycled pid cannot cost a bystander; - redirect stderr before the input redirection when reading /proc//cmdline, so the expected exit-between-glob-and-read race stops printing "No such file or directory"; - pin the bash E2E_ROOTS to both PowerShell mirrors, extracting the `-match` alternation rather than substring-matching the workflow: the whole-file form false-passes, since the roots also appear there as env vars; - correct the PowerShell "same rule" comment, which overstated parity. `rocm ` cannot match `rocm.exe daemon`, and `__engine-serve-http` sits in the root alternation rather than the engine one. Both predate this change and are tracked in EAI-8815. Found by automated review on PR #406. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 16 ++-- .github/workflows/nightly.yml | 16 ++-- scripts/reclaim-gpu.sh | 112 ++++++++++++++++++++++++--- xtask/src/workflow_contract.rs | 98 +++++++++++++++++++++++ 4 files changed, 221 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 926cb7b3c8..bd9f9bbe9e 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -668,11 +668,17 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - # Same rule as scripts/reclaim-gpu.sh, which is the source of truth for - # these markers: an E2E-owned root AND an engine process. Native Windows - # has no bash, so the rule is mirrored here rather than shared. Keep the - # root list in step with the script — `e2e-prewarm` is the shared runtime - # tree the engine binaries actually run from. + # Mirrors the SHAPE of scripts/reclaim-gpu.sh's rule — an E2E-owned root + # AND an engine process — but is not equivalent to it. Native Windows has + # no bash, so the rule is restated here rather than shared. Keep the root + # alternation in step with the script's E2E_ROOTS (`e2e-prewarm` is the + # shared runtime tree the engine binaries actually run from); xtask's + # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. + # + # The engine half diverges in two pre-existing ways, tracked in EAI-8815: + # `rocm ` cannot match Windows' `rocm.exe daemon`, and `__engine-serve-http` + # sits in the root alternation rather than the engine one. Do not read this + # as "same markers as the script" until that is fixed. $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } foreach ($p in $leaked) { diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 31a38e6ef5..0de32af324 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -890,11 +890,17 @@ jobs: - name: Reclaim GPU from stray E2E processes shell: powershell run: | - # Same rule as scripts/reclaim-gpu.sh, which is the source of truth for - # these markers: an E2E-owned root AND an engine process. Native Windows - # has no bash, so the rule is mirrored here rather than shared. Keep the - # root list in step with the script — `e2e-prewarm` is the shared runtime - # tree the engine binaries actually run from. + # Mirrors the SHAPE of scripts/reclaim-gpu.sh's rule — an E2E-owned root + # AND an engine process — but is not equivalent to it. Native Windows has + # no bash, so the rule is restated here rather than shared. Keep the root + # alternation in step with the script's E2E_ROOTS (`e2e-prewarm` is the + # shared runtime tree the engine binaries actually run from); xtask's + # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. + # + # The engine half diverges in two pre-existing ways, tracked in EAI-8815: + # `rocm ` cannot match Windows' `rocm.exe daemon`, and `__engine-serve-http` + # sits in the root alternation rather than the engine one. Do not read this + # as "same markers as the script" until that is fixed. $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } foreach ($p in $leaked) { diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 71d072b0c2..b1a6d6782a 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -95,7 +95,10 @@ select_leaked() { continue fi # A process can exit between the glob and the read; that is not an error. - cmdline="$(tr '\0' ' ' <"${cmdline_file}" 2>/dev/null)" || continue + # Redirect stderr BEFORE the input redirection: the shell applies them left + # to right, so `/dev/null` still lets the shell's own "No such file" + # reach the terminal when the open fails. + cmdline="$(tr '\0' ' ' 2>/dev/null <"${cmdline_file}")" || continue [[ -n "${cmdline}" ]] || continue if [[ -n "${SELFTEST_SCOPE}" ]]; then @@ -138,7 +141,7 @@ select_leaked() { # nothing, and a silent no-op is how the original defect stayed hidden. reclaim() { local dry_run="$1" - local selected pid cmdline + local selected pid cmdline current local killed=0 local waited=0 @@ -176,10 +179,19 @@ reclaim() { # by the kernel when the process dies, so this is what actually frees the card. while IFS=$'\t' read -r pid cmdline; do [[ -n "${pid}" ]] || continue - if process_alive "${pid}"; then - echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" - kill -KILL "${pid}" 2>/dev/null || true + process_alive "${pid}" || continue + # This loop walks the pre-TERM snapshot, and a pid freed during the grace + # window can be handed to an unrelated process. Escalating on the pid alone + # would SIGKILL that bystander, so require the command line to still be the + # one we selected. Not killing a genuine holder is recoverable — the next + # job's reclaim sees it again — where killing a bystander is not. + current="$(tr '\0' ' ' 2>/dev/null <"/proc/${pid}/cmdline")" || continue + if [[ "${current}" != "${cmdline}" ]]; then + echo "reclaim: pid=${pid} was recycled during the grace period, not escalating" + continue fi + echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" + kill -KILL "${pid}" 2>/dev/null || true done <<<"${selected}" echo "reclaim: ${killed} process(es) terminated" @@ -216,8 +228,27 @@ spawn_decoy() { echo $! } +# Spawn a decoy that IGNORES SIGTERM, so the TERM -> grace -> KILL escalation is +# exercised. A plain `cp /bin/sleep` decoy dies on the first TERM and leaves the +# escalation branch unreached, which is how it went untested. +spawn_stubborn_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + # The foreground `sleep 1` children are short-lived and name no E2E root, so + # they are never selected and leave nothing behind once the parent is killed. + cat >"${path}" <<'DECOY' +#!/usr/bin/env bash +trap '' TERM +while :; do sleep 1; done +DECOY + chmod +x "${path}" + "${path}" >/dev/null 2>&1 & + echo $! +} + self_test() { - local tmp prewarm_decoy workload_decoy prewarm_pid workload_pid selected + local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy + local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out local failures=0 # Deliberately NOT under /tmp/rocm-e2e: that prefix is one of the roots the @@ -225,6 +256,9 @@ self_test() { tmp="$(mktemp -d /tmp/reclaim-selftest-XXXXXX)" export RECLAIM_SELFTEST_SCOPE="${tmp}" SELFTEST_SCOPE="${tmp}" + # The stubborn decoy never exits on its own, so the grace loop always runs to + # the ceiling. Keep it short: this is a unit-speed test, not a GPU lane. + TERM_GRACE_SECS=2 # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time trap "rm -rf '${tmp}'" EXIT @@ -232,10 +266,20 @@ self_test() { prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" # A manual-testing serve on a shared runner: an engine, but no E2E root. workload_decoy="${tmp}/workload/manual-serve/llama-server" + # An E2E root with NO engine marker — the suite's own test binary under + # CARGO_TARGET_DIR. This is what the `has_engine` half exists to spare, and + # without it in the fixtures that half can be deleted with the test still green. + harness_decoy="${tmp}/e2e-target/release/deps/e2e-harness" + # Same shape as the pre-warm decoy but ignores SIGTERM, so the escalation this + # script adds is reached. Without it, TERM alone ends every decoy and both the + # SIGKILL block and the `kill -TERM` call can be removed with the test green. + stubborn_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" workload_pid="$(spawn_decoy "${workload_decoy}")" - # Give both decoys a moment to appear in /proc with their full argv. + harness_pid="$(spawn_decoy "${harness_decoy}")" + stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" + # Give the decoys a moment to appear in /proc with their full argv. sleep 1 # 1. Regression guard: the patterns this script replaced could not see a @@ -258,7 +302,7 @@ self_test() { failures=$((failures + 1)) fi - # 3. A manual-testing serve is left alone. + # 3. A manual-testing serve is left alone (engine marker, but no E2E root). if grep -q "^${workload_pid} " <<<"${selected}"; then echo "FAIL: /workload manual serve was selected; reclaim must not touch it" failures=$((failures + 1)) @@ -266,8 +310,25 @@ self_test() { echo "ok: /workload manual serve is not selected" fi - # 4. End to end: reclaim actually kills the leak and spares the bystander. - reclaim 0 >/dev/null + # 4. Both halves are required: an E2E root alone must not select. Deleting the + # `has_engine` requirement makes exactly this check fail and nothing else. + if grep -q "^${harness_pid} " <<<"${selected}"; then + echo "FAIL: E2E test binary was selected; the engine half of the rule is not enforced" + failures=$((failures + 1)) + else + echo "ok: an E2E root without an engine marker is not selected" + fi + + # 5. A process that ignores SIGTERM is still selected. + if grep -q "^${stubborn_pid} " <<<"${selected}"; then + echo "ok: SIGTERM-ignoring pre-warm engine process is selected" + else + echo "FAIL: SIGTERM-ignoring pre-warm engine process was not selected" + failures=$((failures + 1)) + fi + + # 6. End to end: reclaim kills the leaks and spares both bystanders. + reclaim_out="$(reclaim 0)" sleep 1 if process_alive "${prewarm_pid}"; then echo "FAIL: pre-warm engine process survived reclaim" @@ -281,8 +342,37 @@ self_test() { echo "FAIL: /workload manual serve was killed by reclaim" failures=$((failures + 1)) fi + if process_alive "${harness_pid}"; then + echo "ok: E2E test binary survived reclaim" + else + echo "FAIL: E2E test binary was killed by reclaim" + failures=$((failures + 1)) + fi + + # 7. The escalation ran, and ran only where it was needed. Asserting the + # stubborn decoy died covers the SIGKILL block; asserting the ordinary + # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, + # which would otherwise be silently replaceable by any no-op. + if process_alive "${stubborn_pid}"; then + echo "FAIL: SIGTERM-ignoring process survived reclaim; escalation to SIGKILL did not happen" + failures=$((failures + 1)) + else + echo "ok: SIGTERM-ignoring process was escalated to SIGKILL" + fi + if grep -q "pid=${stubborn_pid} ignored SIGTERM" <<<"${reclaim_out}"; then + echo "ok: escalation was reported for the process that ignored SIGTERM" + else + echo "FAIL: no escalation reported for the SIGTERM-ignoring process" + failures=$((failures + 1)) + fi + if grep -q "pid=${prewarm_pid} ignored SIGTERM" <<<"${reclaim_out}"; then + echo "FAIL: ordinary decoy reached SIGKILL escalation; SIGTERM is not being delivered" + failures=$((failures + 1)) + else + echo "ok: ordinary decoy exited on SIGTERM without escalation" + fi - kill -KILL "${workload_pid}" 2>/dev/null || true + kill -KILL "${workload_pid}" "${harness_pid}" "${stubborn_pid}" 2>/dev/null || true if [[ "${failures}" -ne 0 ]]; then echo "reclaim-gpu self-test: ${failures} failure(s)" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 13506fee31..c8388184a6 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2072,4 +2072,102 @@ permissions: let block = " env:\n VALID: one\n MALFORMED\n steps:\n"; let _ = job_mapping(block, "env"); } + + /// The E2E-owned roots declared in `scripts/reclaim-gpu.sh`. + /// + /// Parsed rather than duplicated: a copy here would drift the same way the + /// PowerShell mirrors can, which is the defect this test exists to prevent. + fn reclaim_script_roots() -> Vec { + let p = repo_root().join("scripts/reclaim-gpu.sh"); + let text = std::fs::read_to_string(&p) + .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) + .replace("\r\n", "\n"); + let body = text + .split_once("E2E_ROOTS=(") + .unwrap_or_else(|| panic!("{} must declare E2E_ROOTS=(", p.display())) + .1 + .split_once(')') + .unwrap_or_else(|| panic!("{} has an unterminated E2E_ROOTS array", p.display())) + .0; + let roots: Vec = body + .lines() + .filter_map(|l| { + let l = strip_comment(l).trim(); + l.strip_prefix('\'') + .and_then(|l| l.strip_suffix('\'')) + .map(str::to_owned) + }) + .collect(); + assert!( + !roots.is_empty(), + "{} declared no E2E_ROOTS entries — the parser or the array shape changed", + p.display() + ); + roots + } + + /// The root half of each PowerShell reclaim: the FIRST `-match '…'` + /// alternation on the `Where-Object` line. + /// + /// Extracted rather than substring-matched against the whole file for the + /// reason this module's header gives: every root ALSO appears in these + /// workflows as an env var and in prose, so a `text.contains(root)` check + /// passes even when the alternation itself has lost that root. Confirmed by + /// mutation — deleting `e2e-prewarm` from the alternation left the + /// whole-file form of this test green. + fn powershell_reclaim_root_alternations(text: &str) -> Vec { + text.lines() + .filter(|l| l.contains("Where-Object") && l.contains("-match")) + .map(|l| { + let after = l.split_once("-match").expect("filtered on -match").1; + let body = after + .split_once('\'') + .unwrap_or_else(|| panic!("no opening quote in matcher line: {l}")) + .1; + body.split_once('\'') + .unwrap_or_else(|| panic!("unterminated matcher literal: {l}")) + .0 + .to_owned() + }) + .collect() + } + + /// EAI-8751: native Windows has no bash, so the two PowerShell reclaim steps + /// restate the bash rule instead of sharing it. Nothing else in CI compares + /// the two, so a root added to the script — as `e2e-prewarm` was, to fix a + /// leak that held the card for 16 consecutive jobs — can silently miss the + /// Windows lanes. + /// + /// Only the ROOT half is pinned. The engine half is knowingly divergent on + /// Windows (EAI-8815), so asserting parity there would fail on a difference + /// that is recorded rather than accidental. + #[test] + fn reclaim_roots_are_mirrored_in_the_powershell_reclaims() { + let roots = reclaim_script_roots(); + for workflow in ["e2e-selfhosted.yml", "nightly.yml"] { + let text = read_workflow(workflow); + assert!( + text.contains("Get-CimInstance Win32_Process"), + "{workflow} must keep a PowerShell reclaim step (EAI-8751)" + ); + let alternations = powershell_reclaim_root_alternations(&text); + assert!( + !alternations.is_empty(), + "{workflow} has a PowerShell reclaim but no parsable `-match` alternation \ + — the step's shape changed and this guard went blind (EAI-8751)" + ); + for alternation in &alternations { + for root in &roots { + // `/tmp/rocm-e2e` is POSIX-only; the mirrors match the + // `rocm-e2e` segment, the portable part of the same root. + let needle = root.strip_prefix("/tmp/").unwrap_or(root); + assert!( + alternation.contains(needle), + "{workflow}'s PowerShell reclaim matcher `{alternation}` does not name \ + the E2E root `{needle}` declared in scripts/reclaim-gpu.sh (EAI-8751)" + ); + } + } + } + } } From 12f34d95b1dcd9fb742fa215a1a6682ffc66c1ad Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 18 Sep 2026 10:31:27 +0000 Subject: [PATCH 03/16] ci(e2e): enforce every reclaim root and marker individually (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up review found the fixtures still thin per item: only `e2e-prewarm` and `llama-server` were actually exercised, so removing `e2e-shared`, `/tmp/rocm-e2e` or `e2e-target` from E2E_ROOTS, or `vllm`, `__engine-serve-http` or `rocm daemon` from ENGINE_MARKERS, left the self-test green. Confirmed all six before changing anything. Split the matching rule out of select_leaked into `cmdline_matches_rule` so the self-test can drive each entry directly, and assert both lists against a written-out expectation. The duplication is deliberate: a loop over E2E_ROOTS cannot notice a root DELETED from E2E_ROOTS, which is the drift in question, and the expectation names the PowerShell mirrors that must move with it. All six mutations now fail. The contract test gains the missing direction. It caught a root added to the script and absent from a mirror, but not one removed from the script and left behind in a mirror. `__engine-serve-http` is skipped explicitly, with EAI-8815 named, so fixing that ticket makes the assertion hold rather than surprising whoever does it. Also: - distinguish a process that EXITED during the grace window from one whose pid was recycled. Both took the "was recycled" branch, which described a benign race as a pid collision. `same_selected_process` now returns 0/1/2 and the caller reports each separately, with all three arms asserted; - assert every selected process lies inside the scratch tree before the self-test issues its real kills — SELFTEST_SCOPE was the only thing confining them and nothing checked it held; - say which Windows divergence is a miss (`rocm ` vs `rocm.exe daemon`) and which is an over-match (`__engine-serve-http` in the root alternation). Only the second can cost a bystander, and the old comment listed them neutrally as if equivalent. The recycled-pid guard's CALL SITE remains covered only in the always-escalate direction: making a pid be reused by a different process on demand is not reproducible in a test, so deleting the call site still passes. Recorded in a comment at the assertion rather than left for the next reader to discover. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 12 +- .github/workflows/nightly.yml | 12 +- scripts/reclaim-gpu.sh | 224 ++++++++++++++++++++++----- xtask/src/workflow_contract.rs | 36 ++++- 4 files changed, 233 insertions(+), 51 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index bd9f9bbe9e..5b00403445 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -675,10 +675,14 @@ jobs: # shared runtime tree the engine binaries actually run from); xtask's # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. # - # The engine half diverges in two pre-existing ways, tracked in EAI-8815: - # `rocm ` cannot match Windows' `rocm.exe daemon`, and `__engine-serve-http` - # sits in the root alternation rather than the engine one. Do not read this - # as "same markers as the script" until that is fixed. + # The engine half diverges in two pre-existing ways, tracked in EAI-8815. + # They fail in OPPOSITE directions, and only one can cost a bystander: + # - a MISS: `rocm ` (literal trailing space) cannot match Windows' + # `rocm.exe daemon`, so such a leak is not reclaimed here; + # - an OVER-MATCH: `__engine-serve-http` sits in the root alternation + # rather than the engine one, so on Windows a non-E2E managed serve + # satisfies the root half from that marker alone and can be killed. + # Do not read this as "same markers as the script" until that is fixed. $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } foreach ($p in $leaked) { diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0de32af324..1b15b35525 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -897,10 +897,14 @@ jobs: # shared runtime tree the engine binaries actually run from); xtask's # reclaim_roots_are_mirrored_in_the_powershell_reclaims test pins that half. # - # The engine half diverges in two pre-existing ways, tracked in EAI-8815: - # `rocm ` cannot match Windows' `rocm.exe daemon`, and `__engine-serve-http` - # sits in the root alternation rather than the engine one. Do not read this - # as "same markers as the script" until that is fixed. + # The engine half diverges in two pre-existing ways, tracked in EAI-8815. + # They fail in OPPOSITE directions, and only one can cost a bystander: + # - a MISS: `rocm ` (literal trailing space) cannot match Windows' + # `rocm.exe daemon`, so such a leak is not reclaimed here; + # - an OVER-MATCH: `__engine-serve-http` sits in the root alternation + # rather than the engine one, so on Windows a non-E2E managed serve + # satisfies the root half from that marker alone and can be killed. + # Do not read this as "same markers as the script" until that is fixed. $leaked = Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { $_.CommandLine -match 'rocm-e2e|e2e-prewarm|e2e-shared|e2e-target|__engine-serve-http' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } foreach ($p in $leaked) { diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index b1a6d6782a..328179ef3e 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -83,10 +83,72 @@ process_alive() { [[ "${state}" != "Z" ]] } +# Read a process's command line, NUL-separated in /proc, as a single string. +# Fails when the process is gone or its command line is empty (a kernel thread, +# or a zombie whose argv has already been released). +# +# Redirect stderr BEFORE the input redirection: the shell applies them left to +# right, so `/dev/null` still lets the shell's own "No such file" reach +# the terminal when the open fails. +cmdline_of() { + local cmdline + cmdline="$(tr '\0' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 + [[ -n "${cmdline}" ]] || return 1 + printf '%s' "${cmdline}" +} + +# True when a command line names an E2E root AND an engine marker. +# +# Split out of select_leaked so the self-test can drive every root and every +# marker directly. Exercising them through spawned processes would need one +# process per list entry; leaving them undriven is how `e2e-shared`, `vllm`, +# `__engine-serve-http` and `rocm daemon` came to be removable with the +# self-test still green. +cmdline_matches_rule() { + local cmdline="$1" + local root engine + local has_root=0 + local has_engine=0 + for root in "${E2E_ROOTS[@]}"; do + case "${cmdline}" in + *"${root}"*) + has_root=1 + break + ;; + *) ;; + esac + done + [[ "${has_root}" == 1 ]] || return 1 + for engine in "${ENGINE_MARKERS[@]}"; do + case "${cmdline}" in + *"${engine}"*) + has_engine=1 + break + ;; + *) ;; + esac + done + [[ "${has_engine}" == 1 ]] +} + +# Whether is still running the command line it was selected with. +# 0 same process +# 1 a DIFFERENT command line — the pid was recycled +# 2 gone: exited between the liveness check and this read +# The 1/2 split matters because they are not the same event, and reporting a +# process that simply exited as "recycled" misdescribes a benign race. +same_selected_process() { + local pid="$1" + local expected="$2" + local current + current="$(cmdline_of "${pid}")" || return 2 + [[ "${current}" == "${expected}" ]] +} + # Print "pidcommand line" for every process whose command line names both # an E2E root and an engine marker. select_leaked() { - local cmdline_file pid cmdline root engine has_root has_engine + local cmdline_file pid cmdline for cmdline_file in /proc/[0-9]*/cmdline; do pid="${cmdline_file#/proc/}" pid="${pid%/cmdline}" @@ -95,11 +157,7 @@ select_leaked() { continue fi # A process can exit between the glob and the read; that is not an error. - # Redirect stderr BEFORE the input redirection: the shell applies them left - # to right, so `/dev/null` still lets the shell's own "No such file" - # reach the terminal when the open fails. - cmdline="$(tr '\0' ' ' 2>/dev/null <"${cmdline_file}")" || continue - [[ -n "${cmdline}" ]] || continue + cmdline="$(cmdline_of "${pid}")" || continue if [[ -n "${SELFTEST_SCOPE}" ]]; then case "${cmdline}" in @@ -108,29 +166,7 @@ select_leaked() { esac fi - has_root=0 - for root in "${E2E_ROOTS[@]}"; do - case "${cmdline}" in - *"${root}"*) - has_root=1 - break - ;; - *) ;; - esac - done - [[ "${has_root}" == 1 ]] || continue - - has_engine=0 - for engine in "${ENGINE_MARKERS[@]}"; do - case "${cmdline}" in - *"${engine}"*) - has_engine=1 - break - ;; - *) ;; - esac - done - [[ "${has_engine}" == 1 ]] || continue + cmdline_matches_rule "${cmdline}" || continue printf '%s\t%s\n' "${pid}" "${cmdline}" done @@ -141,7 +177,7 @@ select_leaked() { # nothing, and a silent no-op is how the original defect stayed hidden. reclaim() { local dry_run="$1" - local selected pid cmdline current + local selected pid cmdline rc local killed=0 local waited=0 @@ -185,13 +221,16 @@ reclaim() { # would SIGKILL that bystander, so require the command line to still be the # one we selected. Not killing a genuine holder is recoverable — the next # job's reclaim sees it again — where killing a bystander is not. - current="$(tr '\0' ' ' 2>/dev/null <"/proc/${pid}/cmdline")" || continue - if [[ "${current}" != "${cmdline}" ]]; then - echo "reclaim: pid=${pid} was recycled during the grace period, not escalating" - continue - fi - echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" - kill -KILL "${pid}" 2>/dev/null || true + rc=0 + same_selected_process "${pid}" "${cmdline}" || rc=$? + case "${rc}" in + 0) + echo "reclaim: pid=${pid} ignored SIGTERM after ${TERM_GRACE_SECS}s, sending SIGKILL" + kill -KILL "${pid}" 2>/dev/null || true + ;; + 2) echo "reclaim: pid=${pid} exited during the grace period" ;; + *) echo "reclaim: pid=${pid} was recycled during the grace period, not escalating" ;; + esac done <<<"${selected}" echo "reclaim: ${killed} process(es) terminated" @@ -246,11 +285,71 @@ DECOY echo $! } +# Every root and every marker, asserted individually against the rule. +# +# The expectation is written out rather than derived from the arrays: a loop +# over E2E_ROOTS cannot notice a root DELETED from E2E_ROOTS, which is exactly +# the drift that left `e2e-shared`, `vllm`, `__engine-serve-http` and +# `rocm daemon` removable with the self-test green. The duplication is the +# point — changing either list must be a deliberate edit in two places, one of +# which names the PowerShell mirrors that also have to move. +assert_rule_covers_every_list_entry() { + local failures=0 + local root engine + local expected_roots=('/tmp/rocm-e2e' 'e2e-shared' 'e2e-prewarm' 'e2e-target') + local expected_markers=('llama-server' 'vllm' '__engine-serve-http' 'rocm daemon') + + if [[ "$(printf '%s\n' "${E2E_ROOTS[@]}")" != "$(printf '%s\n' "${expected_roots[@]}")" ]]; then + echo "FAIL: E2E_ROOTS changed — update this expectation AND both PowerShell mirrors" + echo " script: ${E2E_ROOTS[*]}" + echo " expected: ${expected_roots[*]}" + failures=$((failures + 1)) + fi + if [[ "$(printf '%s\n' "${ENGINE_MARKERS[@]}")" != "$(printf '%s\n' "${expected_markers[@]}")" ]]; then + echo "FAIL: ENGINE_MARKERS changed — update this expectation AND both PowerShell mirrors" + echo " script: ${ENGINE_MARKERS[*]}" + echo " expected: ${expected_markers[*]}" + failures=$((failures + 1)) + fi + + for root in "${expected_roots[@]}"; do + if ! cmdline_matches_rule "${root}/bin/llama-server --model m"; then + echo "FAIL: root '${root}' with an engine marker does not match the rule" + failures=$((failures + 1)) + fi + if cmdline_matches_rule "${root}/bin/e2e-harness --exact"; then + echo "FAIL: root '${root}' matched with NO engine marker; the AND rule is broken" + failures=$((failures + 1)) + fi + done + + for engine in "${expected_markers[@]}"; do + if ! cmdline_matches_rule "e2e-prewarm/bin/${engine} --serve"; then + echo "FAIL: marker '${engine}' under an E2E root does not match the rule" + failures=$((failures + 1)) + fi + if cmdline_matches_rule "workload/bin/${engine} --serve"; then + echo "FAIL: marker '${engine}' matched with NO E2E root; a manual serve is not safe" + failures=$((failures + 1)) + fi + done + + if [[ "${failures}" -eq 0 ]]; then + echo "ok: every E2E root and every engine marker is individually enforced" + fi + return "${failures}" +} + self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out + local escapee_pid escapee_cmd guard_rc + local list_failures=0 local failures=0 + assert_rule_covers_every_list_entry || list_failures=$? + failures=$((failures + list_failures)) + # Deliberately NOT under /tmp/rocm-e2e: that prefix is one of the roots the # old patterns did match, which would mask the regression this guards. tmp="$(mktemp -d /tmp/reclaim-selftest-XXXXXX)" @@ -327,7 +426,56 @@ self_test() { failures=$((failures + 1)) fi - # 6. End to end: reclaim kills the leaks and spares both bystanders. + # 6. Containment: self_test issues REAL kills, and SELFTEST_SCOPE is the only + # thing keeping them inside the scratch tree. Assert that before killing + # rather than trusting it — this step runs on a hosted ephemeral lane + # today, but nothing in the script stops it being run anywhere else. + while IFS=$'\t' read -r escapee_pid escapee_cmd; do + [[ -n "${escapee_pid}" ]] || continue + case "${escapee_cmd}" in + *"${tmp}"*) ;; + *) + echo "FAIL: selection escaped the scratch tree: pid=${escapee_pid} cmd=${escapee_cmd}" + failures=$((failures + 1)) + ;; + esac + done <<<"${selected}" + if [[ "${failures}" -eq "${list_failures}" ]]; then + echo "ok: every selected process lies inside the self-test scratch tree" + fi + + # 7. The escalation guard's comparison, in all three directions. + # + # NOTE: only the COMPARISON is covered. The guard's call site is exercised + # solely in the always-escalate direction, because making a pid be reused + # by a different process on demand is not reproducible in a test — so + # deleting the call site still passes the self-test. Said plainly rather + # than implied by a green run. + if same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")"; then + echo "ok: escalation guard accepts an unchanged command line" + else + echo "FAIL: escalation guard rejected an unchanged command line; nothing would escalate" + failures=$((failures + 1)) + fi + guard_rc=0 + same_selected_process "${stubborn_pid}" "/some/other/process --unrelated" || guard_rc=$? + if [[ "${guard_rc}" == 1 ]]; then + echo "ok: escalation guard rejects a recycled pid" + else + echo "FAIL: escalation guard did not report a changed command line as recycled (rc=${guard_rc})" + failures=$((failures + 1)) + fi + guard_rc=0 + # A pid above /proc/sys/kernel/pid_max cannot exist, so this is the "gone" arm. + same_selected_process 2147483647 "anything" || guard_rc=$? + if [[ "${guard_rc}" == 2 ]]; then + echo "ok: escalation guard reports a departed process as gone, not recycled" + else + echo "FAIL: escalation guard conflated a departed process with a recycled one (rc=${guard_rc})" + failures=$((failures + 1)) + fi + + # 8. End to end: reclaim kills the leaks and spares both bystanders. reclaim_out="$(reclaim 0)" sleep 1 if process_alive "${prewarm_pid}"; then diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c8388184a6..36d5295875 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2156,17 +2156,43 @@ permissions: "{workflow} has a PowerShell reclaim but no parsable `-match` alternation \ — the step's shape changed and this guard went blind (EAI-8751)" ); + // `/tmp/rocm-e2e` is POSIX-only; the mirrors match the `rocm-e2e` + // segment, the portable part of the same root. + let expected: Vec<&str> = roots + .iter() + .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) + .collect(); + for alternation in &alternations { - for root in &roots { - // `/tmp/rocm-e2e` is POSIX-only; the mirrors match the - // `rocm-e2e` segment, the portable part of the same root. - let needle = root.strip_prefix("/tmp/").unwrap_or(root); + let present: Vec<&str> = alternation.split('|').collect(); + + // Script -> mirror: a root added to the script must reach Windows. + for needle in &expected { assert!( - alternation.contains(needle), + present.contains(needle), "{workflow}'s PowerShell reclaim matcher `{alternation}` does not name \ the E2E root `{needle}` declared in scripts/reclaim-gpu.sh (EAI-8751)" ); } + + // Mirror -> script: and a root REMOVED from the script must not be + // left behind here. Without this direction the guard is one-way, + // which is how the lists drifted in the first place. + for token in &present { + // Known divergence, not drift: `__engine-serve-http` is an + // ENGINE marker that sits in this root alternation, so on + // Windows it over-matches. Tracked in EAI-8815 — when that is + // fixed, delete this arm and the assertion below will hold. + if *token == "__engine-serve-http" { + continue; + } + assert!( + expected.contains(token), + "{workflow}'s PowerShell reclaim matcher `{alternation}` names `{token}`, \ + which is not an E2E root in scripts/reclaim-gpu.sh — remove it here too, \ + or add it there (EAI-8751)" + ); + } } } } From 3390dc8746f70e9947d9fb87a8516617548bc883 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 21 Sep 2026 14:20:16 +0000 Subject: [PATCH 04/16] ci(e2e): close review round on the reclaim self-test (EAI-8751) Exercise both routes into the escalation guard's "gone" result (a failed /proc open and an empty read from a real zombie decoy), so cmdline_of's emptiness check stays covered. Have the superseded-pattern regression guard read the decoys' own command lines instead of asking pgrep -f a machine-wide question that also matches the invoking shell, and fail loudly instead of passing vacuously when a probe can't be read. Make the containment check report its own failure count instead of inheriting the running total, which suppressed its "ok" line whenever an earlier check had failed. Reap the pre-warm decoy on a failed run instead of orphaning it for 300s, and fix a mis-numbered step comment. In xtask, assert the __engine-serve-http divergence exemption fires, so it fails (demanding its own deletion) once EAI-8815 closes the divergence instead of rotting into dead code. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 114 ++++++++++++++++++++++++++++++--- xtask/src/workflow_contract.rs | 19 ++++++ 2 files changed, 124 insertions(+), 9 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 328179ef3e..f502d168cf 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -267,6 +267,46 @@ spawn_decoy() { echo $! } +# Spawn a process that becomes a ZOMBIE and stays one, plus the keeper holding +# it in that state. Echoes " "; the zombie pid is empty +# if it could not be produced. +# +# A zombie is the second way into cmdline_of's failure path and the only one +# that reaches its emptiness check: /proc//cmdline still OPENS for a +# zombie, it just reads zero bytes, because the kernel has released the argv +# while the process table entry remains. +spawn_zombie() { + local dir="$1" + local keeper_pid zombie_pid state + local waited=0 + mkdir -p "${dir}" + # The subshell starts a short-lived child and then `exec`s a long sleep, so + # the parent that would reap it is replaced by a process that never calls + # wait(). The child therefore stays a zombie for as long as the keeper lives. + # Bash reaps its OWN background children, which is why this needs the exec. + ( + sleep 0.1 & + echo $! >"${dir}/zombie.pid" + exec sleep 300 + ) >/dev/null 2>&1 & + keeper_pid=$! + # Wait for the child to actually reach state Z rather than assuming it has. + while [[ "${waited}" -lt 50 ]]; do + if [[ -s "${dir}/zombie.pid" ]]; then + zombie_pid="$(cat "${dir}/zombie.pid")" + state="$(sed 's/.*) //' "/proc/${zombie_pid}/stat" 2>/dev/null | cut -d' ' -f1)" || state='' + [[ "${state}" == "Z" ]] && break + fi + sleep 0.1 + waited=$((waited + 1)) + done + # Sentinel, not an empty field: `read` skips leading whitespace, so echoing an + # empty first field would shift the KEEPER's pid into the caller's zombie_pid + # and make the "could not produce one" branch unreachable. + [[ "${state:-}" == "Z" ]] || zombie_pid='none' + echo "${zombie_pid} ${keeper_pid}" +} + # Spawn a decoy that IGNORES SIGTERM, so the TERM -> grace -> KILL escalation is # exercised. A plain `cp /bin/sleep` decoy dies on the first TERM and leaves the # escalation branch unreached, which is how it went untested. @@ -343,7 +383,11 @@ assert_rule_covers_every_list_entry() { self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out - local escapee_pid escapee_cmd guard_rc + local escapee_pid escapee_cmd guard_rc probe_cmd + local zombie_pid zombie_keeper_pid + local superseded_hit=0 + local probe_failures=0 + local containment_failures=0 local list_failures=0 local failures=0 @@ -378,17 +422,39 @@ self_test() { workload_pid="$(spawn_decoy "${workload_decoy}")" harness_pid="$(spawn_decoy "${harness_decoy}")" stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" + read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" # Give the decoys a moment to appear in /proc with their full argv. sleep 1 # 1. Regression guard: the patterns this script replaced could not see a # pre-warm engine process. If this ever matches, the decoy stopped being # representative and the rest of the self-test proves nothing. - if pgrep -f '/tmp/rocm-e2e.*llama-server' >/dev/null 2>&1 || - pgrep -f 'e2e-shared.*llama-server' >/dev/null 2>&1; then + # + # Asked of the decoys' own command lines, not of the machine. The obvious + # `pgrep -f ''` also matches the shell that INVOKED it whenever + # that shell's own command line contains the pattern, so it reports a + # match with no such process alive; on a shared self-hosted runner an + # unrelated process would fail it too. Both are false FAILs in the check + # whose entire job is to say "the decoy is unrepresentative". + for probe_cmd in "$(cmdline_of "${prewarm_pid}")" "$(cmdline_of "${stubborn_pid}")"; do + # An unreadable probe would leave every pattern unmatched and print "ok" + # having tested nothing. Same reason the zombie arm below fails loudly + # rather than skipping: a check that cannot run must not report a pass. + if [[ -z "${probe_cmd}" ]]; then + echo "FAIL: could not read a decoy's command line; the regression guard tested nothing" + probe_failures=$((probe_failures + 1)) + continue + fi + if [[ "${probe_cmd}" =~ /tmp/rocm-e2e.*llama-server ]] || + [[ "${probe_cmd}" =~ e2e-shared.*llama-server ]]; then + superseded_hit=1 + fi + done + failures=$((failures + probe_failures)) + if [[ "${superseded_hit}" == 1 ]]; then echo "FAIL: superseded patterns matched the pre-warm decoy; decoy is unrepresentative" failures=$((failures + 1)) - else + elif [[ "${probe_failures}" -eq 0 ]]; then echo "ok: superseded patterns do not match a pre-warm engine process (the defect)" fi @@ -436,15 +502,20 @@ self_test() { *"${tmp}"*) ;; *) echo "FAIL: selection escaped the scratch tree: pid=${escapee_pid} cmd=${escapee_cmd}" - failures=$((failures + 1)) + containment_failures=$((containment_failures + 1)) ;; esac done <<<"${selected}" - if [[ "${failures}" -eq "${list_failures}" ]]; then + failures=$((failures + containment_failures)) + # Counted on its own rather than off the running total: gating this line on + # the total suppressed it whenever an EARLIER check had failed — which is + # precisely the run whose output someone is reading. + if [[ "${containment_failures}" -eq 0 ]]; then echo "ok: every selected process lies inside the self-test scratch tree" fi - # 7. The escalation guard's comparison, in all three directions. + # 7. The escalation guard's comparison, in all three directions — and for + # "gone", by BOTH routes into it. # # NOTE: only the COMPARISON is covered. The guard's call site is exercised # solely in the always-escalate direction, because making a pid be reused @@ -474,6 +545,26 @@ self_test() { echo "FAIL: escalation guard conflated a departed process with a recycled one (rc=${guard_rc})" failures=$((failures + 1)) fi + # The other route into "gone": the open SUCCEEDS and reads nothing. The arm + # above exercises only a failed OPEN, so without this one `cmdline_of`'s + # emptiness check can be deleted with the self-test still green — restoring + # the "was recycled" mislabel for a process that merely exited, which is the + # race the guard exists to describe correctly. + if [[ "${zombie_pid}" != "none" ]]; then + guard_rc=0 + same_selected_process "${zombie_pid}" "anything" || guard_rc=$? + if [[ "${guard_rc}" == 2 ]]; then + echo "ok: escalation guard reports an argv-less zombie as gone, not recycled" + else + echo "FAIL: zombie with an empty command line was not reported as gone (rc=${guard_rc})" + failures=$((failures + 1)) + fi + else + # Failing rather than skipping: an arm that silently does not run is the + # exact defect this check was added to close. + echo "FAIL: could not produce a zombie decoy; the empty-cmdline arm went untested" + failures=$((failures + 1)) + fi # 8. End to end: reclaim kills the leaks and spares both bystanders. reclaim_out="$(reclaim 0)" @@ -497,7 +588,7 @@ self_test() { failures=$((failures + 1)) fi - # 7. The escalation ran, and ran only where it was needed. Asserting the + # 9. The escalation ran, and ran only where it was needed. Asserting the # stubborn decoy died covers the SIGKILL block; asserting the ordinary # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, # which would otherwise be silently replaceable by any no-op. @@ -520,7 +611,12 @@ self_test() { echo "ok: ordinary decoy exited on SIGTERM without escalation" fi - kill -KILL "${workload_pid}" "${harness_pid}" "${stubborn_pid}" 2>/dev/null || true + # Includes prewarm_pid: on a GREEN run reclaim has already killed it, but on a + # FAILED run it was not selected, and it would otherwise outlive the scratch + # tree for its full 300s as an orphan. Killing the zombie's keeper lets init + # reap the zombie itself. + kill -KILL "${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" \ + "${zombie_keeper_pid}" 2>/dev/null || true if [[ "${failures}" -ne 0 ]]; then echo "reclaim-gpu self-test: ${failures} failure(s)" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 36d5295875..8deb62dad1 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2109,6 +2109,11 @@ permissions: /// The root half of each PowerShell reclaim: the FIRST `-match '…'` /// alternation on the `Where-Object` line. /// + /// This cannot tell the reclaim's own matcher from any other + /// `Where-Object … -match` line. Each of these workflows has exactly one + /// today, so every alternation returned IS a reclaim matcher; add a second, + /// unrelated one and the caller's assertions would be applied to it too. + /// /// Extracted rather than substring-matched against the whole file for the /// reason this module's header gives: every root ALSO appears in these /// workflows as an env var and in prose, so a `text.contains(root)` check @@ -2165,6 +2170,7 @@ permissions: for alternation in &alternations { let present: Vec<&str> = alternation.split('|').collect(); + let mut divergence_seen = false; // Script -> mirror: a root added to the script must reach Windows. for needle in &expected { @@ -2184,6 +2190,7 @@ permissions: // Windows it over-matches. Tracked in EAI-8815 — when that is // fixed, delete this arm and the assertion below will hold. if *token == "__engine-serve-http" { + divergence_seen = true; continue; } assert!( @@ -2193,6 +2200,18 @@ permissions: or add it there (EAI-8751)" ); } + + // An exemption nothing asserts is an exemption that rots: once + // EAI-8815 moves `__engine-serve-http` out of the root + // alternation, the arm above stops firing and would sit here + // forever as dead code exempting nothing. Fail instead, so the + // fix is told to finish the job. + assert!( + divergence_seen, + "{workflow}'s PowerShell reclaim matcher `{alternation}` no longer names \ + `__engine-serve-http`, so the EAI-8815 divergence looks fixed — delete the \ + exemption arm in this test, which is now dead code (EAI-8751)" + ); } } } From 4bfdb1c3ab254858297f348b8e02dee9be793cbf Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 21 Sep 2026 15:08:51 +0000 Subject: [PATCH 05/16] ci(e2e): re-verify pid identity before SIGTERM too (EAI-8751) reclaim() re-checked a pid's command line before the SIGKILL escalation but signalled SIGTERM straight off the select_leaked snapshot. A pid freed between the /proc scan and the signal can be reissued to an unrelated process, and a bystander is just as dead from TERM as from KILL, so the same guard now applies to both, reporting recycled and exited pids separately. The self-test's containment check scored an escaped selection as a failure and then fell through into the step that issues real signals, so the one regression it exists to catch would have made it perform the damage it guards against. It is now a gate: it reports, kills its decoys, and returns before anything is signalled. A decoy_pids array replaces the duplicated cleanup list. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index f502d168cf..2d44c438f8 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -192,6 +192,24 @@ reclaim() { if [[ "${dry_run}" == 1 ]]; then echo "reclaim: would terminate pid=${pid} cmd=${cmdline}" else + # Same guard as the escalation below, and for the same reason. The window + # is shorter here — no grace period — but it is not zero: select_leaked + # walks the whole of /proc, so a pid can be freed and reissued between + # the scan that selected it and this signal. A bystander is just as dead + # from TERM as from KILL, so the check cannot be reserved for the latter. + rc=0 + same_selected_process "${pid}" "${cmdline}" || rc=$? + case "${rc}" in + 0) ;; + 2) + echo "reclaim: pid=${pid} exited before it could be terminated" + continue + ;; + *) + echo "reclaim: pid=${pid} was recycled before it could be terminated, not signalling" + continue + ;; + esac echo "reclaim: terminating pid=${pid} cmd=${cmdline}" kill -TERM "${pid}" 2>/dev/null || true fi @@ -385,6 +403,7 @@ self_test() { local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out local escapee_pid escapee_cmd guard_rc probe_cmd local zombie_pid zombie_keeper_pid + local decoy_pids local superseded_hit=0 local probe_failures=0 local containment_failures=0 @@ -423,6 +442,11 @@ self_test() { harness_pid="$(spawn_decoy "${harness_decoy}")" stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" + # Every process this function spawned that can still be signalled, so cleanup + # is one list rather than a line kept in step at each early return. The zombie + # itself is absent deliberately: it is already dead, and killing its keeper is + # what lets init reap it. + decoy_pids=("${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" "${zombie_keeper_pid}") # Give the decoys a moment to appear in /proc with their full argv. sleep 1 @@ -512,6 +536,17 @@ self_test() { # precisely the run whose output someone is reading. if [[ "${containment_failures}" -eq 0 ]]; then echo "ok: every selected process lies inside the self-test scratch tree" + else + # A GATE, not a score. Check 8 below calls the real `reclaim 0`, which + # sends real signals to whatever select_leaked returns at that moment. If + # containment has just failed, that selection reaches outside this scratch + # tree — the exact accident this check exists to prevent — so scoring it + # and carrying on would let the self-test do the damage it is guarding + # against. Stop here instead, while nothing has been signalled yet. + echo "FAIL: containment breached; refusing to run the real reclaim" + kill -KILL "${decoy_pids[@]}" 2>/dev/null || true + echo "reclaim-gpu self-test: ${failures} failure(s)" + return 1 fi # 7. The escalation guard's comparison, in all three directions — and for @@ -615,8 +650,7 @@ self_test() { # FAILED run it was not selected, and it would otherwise outlive the scratch # tree for its full 300s as an orphan. Killing the zombie's keeper lets init # reap the zombie itself. - kill -KILL "${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" \ - "${zombie_keeper_pid}" 2>/dev/null || true + kill -KILL "${decoy_pids[@]}" 2>/dev/null || true if [[ "${failures}" -ne 0 ]]; then echo "reclaim-gpu self-test: ${failures} failure(s)" From df8bc9ad7a2239a752ea831b78c8353032c0083b Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 21 Sep 2026 15:35:51 +0000 Subject: [PATCH 06/16] ci(e2e): give the reclaim self-test a real containment case (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scope-containment check filtered candidates through SELFTEST_SCOPE, which is ${tmp}, then asserted every selected entry was inside ${tmp} — re-deriving its answer from the filter it claimed to verify. Every fixture lived inside the scope, so it could never fail; deleting the scope filter left the check green. Add a second scratch tree and a bystander decoy that matches the reclaim rule in full but lies outside the scope, so the filter is the only thing that can exclude it, assert the bystander is running and still matches the rule so it can't quietly stop being representative, and confirm in check 8 that it survives the real reclaim. Also move the EXIT trap to right after the first mktemp and widen it after the second, so a failed second mktemp under set -e can't abort with the first scratch dir already created and nothing to remove it, and fix the escalation-guard comment to say two call sites now that a prior commit added the pre-KILL one. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 63 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 2d44c438f8..842a7996d7 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -401,6 +401,7 @@ assert_rule_covers_every_list_entry() { self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out + local outside outside_decoy outside_pid outside_cmd local escapee_pid escapee_cmd guard_rc probe_cmd local zombie_pid zombie_keeper_pid local decoy_pids @@ -416,13 +417,23 @@ self_test() { # Deliberately NOT under /tmp/rocm-e2e: that prefix is one of the roots the # old patterns did match, which would mask the regression this guards. tmp="$(mktemp -d /tmp/reclaim-selftest-XXXXXX)" + # Armed the moment there is something to remove, and widened below once the + # second tree exists. The `mktemp` that follows can fail, and under `set -e` + # that aborts the function — with `tmp` already on disk and, if the trap were + # installed only afterwards, nothing left to clean it up. + # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time + trap "rm -rf '${tmp}'" EXIT + # A SECOND tree, outside the scope, for the bystander decoy below. Everything + # reachable through SELFTEST_SCOPE is inside `tmp` by construction, so a + # negative case for the scope filter cannot live there. + outside="$(mktemp -d /tmp/reclaim-selftest-out-XXXXXX)" export RECLAIM_SELFTEST_SCOPE="${tmp}" SELFTEST_SCOPE="${tmp}" # The stubborn decoy never exits on its own, so the grace loop always runs to # the ceiling. Keep it short: this is a unit-speed test, not a GPU lane. TERM_GRACE_SECS=2 - # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time - trap "rm -rf '${tmp}'" EXIT + # shellcheck disable=SC2064 # expand the paths now, at trap definition time + trap "rm -rf '${tmp}' '${outside}'" EXIT # The real shape: lemonade's engine binary inside the shared pre-warm runtime. prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" @@ -436,17 +447,23 @@ self_test() { # script adds is reached. Without it, TERM alone ends every decoy and both the # SIGKILL block and the `kill -TERM` call can be removed with the test green. stubborn_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" + # Matches the rule in full — an E2E root AND an engine marker — but lies + # OUTSIDE the scope. It is the only fixture the SELFTEST_SCOPE filter can be + # observed doing anything to, and therefore the only reason check 6 can fail. + outside_decoy="${outside}/e2e-prewarm-bystander/bin/llama-server" prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" workload_pid="$(spawn_decoy "${workload_decoy}")" harness_pid="$(spawn_decoy "${harness_decoy}")" stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" + outside_pid="$(spawn_decoy "${outside_decoy}")" read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" # Every process this function spawned that can still be signalled, so cleanup # is one list rather than a line kept in step at each early return. The zombie # itself is absent deliberately: it is already dead, and killing its keeper is # what lets init reap it. - decoy_pids=("${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" "${zombie_keeper_pid}") + decoy_pids=("${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" + "${outside_pid}" "${zombie_keeper_pid}") # Give the decoys a moment to appear in /proc with their full argv. sleep 1 @@ -520,6 +537,27 @@ self_test() { # thing keeping them inside the scratch tree. Assert that before killing # rather than trusting it — this step runs on a hosted ephemeral lane # today, but nothing in the script stops it being run anywhere else. + # The bystander decoy is what gives this check teeth. select_leaked filters + # every candidate against SELFTEST_SCOPE, and SELFTEST_SCOPE *is* ${tmp}, so + # the loop below — "is every selected entry inside ${tmp}?" — re-derives its + # answer from the very filter it claims to be checking, and cannot fail on + # any fixture that lives inside the scope. Deleting the filter outright left + # this check green. A rule-matching process OUTSIDE the scope is the only + # thing the filter can be caught NOT doing its job on. + outside_cmd="$(cmdline_of "${outside_pid}")" || outside_cmd='' + if [[ -z "${outside_cmd}" ]]; then + echo "FAIL: bystander decoy is not running; containment has no negative case" + containment_failures=$((containment_failures + 1)) + elif ! cmdline_matches_rule "${outside_cmd}"; then + # Asserted, not assumed: if the bystander stopped matching the rule it + # would be excluded for that reason instead of by the scope filter, and + # the negative case below would pass while proving nothing. + echo "FAIL: bystander decoy no longer matches the rule; containment proves nothing" + containment_failures=$((containment_failures + 1)) + elif grep -q "^${outside_pid} " <<<"${selected}"; then + echo "FAIL: a rule-matching process outside the scratch tree was selected" + containment_failures=$((containment_failures + 1)) + fi while IFS=$'\t' read -r escapee_pid escapee_cmd; do [[ -n "${escapee_pid}" ]] || continue case "${escapee_cmd}" in @@ -552,11 +590,12 @@ self_test() { # 7. The escalation guard's comparison, in all three directions — and for # "gone", by BOTH routes into it. # - # NOTE: only the COMPARISON is covered. The guard's call site is exercised - # solely in the always-escalate direction, because making a pid be reused - # by a different process on demand is not reproducible in a test — so - # deleting the call site still passes the self-test. Said plainly rather - # than implied by a green run. + # NOTE: only the COMPARISON is covered. The guard now has TWO call sites — + # the pre-TERM check and the pre-KILL escalation — and both are exercised + # solely in the always-proceed direction, because making a pid be reused by + # a different process on demand is not reproducible in a test. Deleting + # either call site still passes the self-test. Said plainly rather than + # implied by a green run. if same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")"; then echo "ok: escalation guard accepts an unchanged command line" else @@ -622,6 +661,14 @@ self_test() { echo "FAIL: E2E test binary was killed by reclaim" failures=$((failures + 1)) fi + # The bystander matches the rule in full, so only the scope kept it out of the + # selection. Its survival is the end-to-end form of check 6. + if process_alive "${outside_pid}"; then + echo "ok: rule-matching process outside the scratch tree survived reclaim" + else + echo "FAIL: reclaim killed a rule-matching process outside the scratch tree" + failures=$((failures + 1)) + fi # 9. The escalation ran, and ran only where it was needed. Asserting the # stubborn decoy died covers the SIGKILL block; asserting the ordinary From ca136ca7f2b5171d90dd1946c0e55cbc298bfff3 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 21 Sep 2026 16:00:32 +0000 Subject: [PATCH 07/16] ci(e2e): correct the bystander count in a self-test comment (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check 8 spares three bystanders since the out-of-scope decoy was added — the manual serve, the harness binary and that decoy — but its header still said two. Comment only; no behaviour change. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 842a7996d7..6f71068988 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -640,7 +640,8 @@ self_test() { failures=$((failures + 1)) fi - # 8. End to end: reclaim kills the leaks and spares both bystanders. + # 8. End to end: reclaim kills the leaks and spares all three bystanders — + # the manual serve, the harness binary, and the out-of-scope decoy. reclaim_out="$(reclaim 0)" sleep 1 if process_alive "${prewarm_pid}"; then From f3ca541d8f0a690a960f51d467b322bdc87c6f56 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 22 Sep 2026 11:10:56 +0000 Subject: [PATCH 08/16] ci(e2e): close five review findings on the reclaim self-test (EAI-8751) Cover both pid-recycling guard call sites (pre-TERM and pre-KILL) by overriding the identity comparison inside a command substitution subshell for one real reclaim run, so a broken guard would kill decoys instead of leaving the self-test green. Split the empty- selection containment failure from the scope-escape counter so it reports its own cause instead of tripping that gate. Fix spawn_zombie's header: the sentinel is the string "none", not an empty field. Widen the EXIT trap to cover the second scratch tree in the statement right after it is created, and drop the duplicate trap. In xtask, track the divergence exemption per workflow instead of per alternation so an unrelated PowerShell match line can't fail it spuriously. Renumber self-test steps 1..10. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 83 ++++++++++++++++++++++++++++------ xtask/src/workflow_contract.rs | 32 +++++++------ 2 files changed, 87 insertions(+), 28 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 6f71068988..87b9167f07 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -286,8 +286,9 @@ spawn_decoy() { } # Spawn a process that becomes a ZOMBIE and stays one, plus the keeper holding -# it in that state. Echoes " "; the zombie pid is empty -# if it could not be produced. +# it in that state. Echoes " "; the zombie pid is the +# literal string `none` if one could not be produced — NOT an empty field, for +# the reason given at the sentinel itself. Test `!= "none"`, never `-z`. # # A zombie is the second way into cmdline_of's failure path and the only one # that reaches its emptiness check: /proc//cmdline still OPENS for a @@ -402,6 +403,7 @@ self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out local outside outside_decoy outside_pid outside_cmd + local forced_out guard_pid local escapee_pid escapee_cmd guard_rc probe_cmd local zombie_pid zombie_keeper_pid local decoy_pids @@ -427,13 +429,16 @@ self_test() { # reachable through SELFTEST_SCOPE is inside `tmp` by construction, so a # negative case for the scope filter cannot live there. outside="$(mktemp -d /tmp/reclaim-selftest-out-XXXXXX)" + # Widened in the statement immediately after the one that created it, for the + # same reason the narrow trap above exists: anything fallible in between is a + # window where a tree is on disk with nothing arranged to remove it. + # shellcheck disable=SC2064 # expand both paths now, at trap definition time + trap "rm -rf '${tmp}' '${outside}'" EXIT export RECLAIM_SELFTEST_SCOPE="${tmp}" SELFTEST_SCOPE="${tmp}" # The stubborn decoy never exits on its own, so the grace loop always runs to # the ceiling. Keep it short: this is a unit-speed test, not a GPU lane. TERM_GRACE_SECS=2 - # shellcheck disable=SC2064 # expand the paths now, at trap definition time - trap "rm -rf '${tmp}' '${outside}'" EXIT # The real shape: lemonade's engine binary inside the shared pre-warm runtime. prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" @@ -544,6 +549,20 @@ self_test() { # any fixture that lives inside the scope. Deleting the filter outright left # this check green. A rule-matching process OUTSIDE the scope is the only # thing the filter can be caught NOT doing its job on. + # An empty selection satisfies the loop below trivially, so without this the + # "ok:" line would report containment verified on a run where nothing was + # examined. Checks 2 and 5 already fail such a run, so this is not a false + # green — but the evidence line is read by whoever is diagnosing that run. + # + # Counted apart from containment_failures on purpose: that counter arms the + # gate below, whose message and early return are specifically about a + # selection reaching OUTSIDE the scratch tree. An empty selection is the + # opposite failure and signals nothing at all, so routing it through that + # gate would report a false cause and cut the run short of checks 7-10. + if [[ -z "${selected}" ]]; then + echo "FAIL: selection was empty; containment had nothing to examine" + failures=$((failures + 1)) + fi outside_cmd="$(cmdline_of "${outside_pid}")" || outside_cmd='' if [[ -z "${outside_cmd}" ]]; then echo "FAIL: bystander decoy is not running; containment has no negative case" @@ -590,12 +609,9 @@ self_test() { # 7. The escalation guard's comparison, in all three directions — and for # "gone", by BOTH routes into it. # - # NOTE: only the COMPARISON is covered. The guard now has TWO call sites — - # the pre-TERM check and the pre-KILL escalation — and both are exercised - # solely in the always-proceed direction, because making a pid be reused by - # a different process on demand is not reproducible in a test. Deleting - # either call site still passes the self-test. Said plainly rather than - # implied by a green run. + # The COMPARISON is covered here; check 8 covers the two CALL SITES by + # forcing the verdict, because a pid cannot be made to be reused by a + # different process on demand. if same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")"; then echo "ok: escalation guard accepts an unchanged command line" else @@ -640,7 +656,44 @@ self_test() { failures=$((failures + 1)) fi - # 8. End to end: reclaim kills the leaks and spares all three bystanders — + # 8. The guard's two CALL SITES, by forcing the verdict they act on. + # + # A pid cannot be made to be reused by a different process on demand, so + # until now both call sites were exercised only in the always-proceed + # direction and either could be deleted with the self-test green. Overriding + # the comparison for one real `reclaim` run reaches them: with every verdict + # "recycled", a correct reclaim signals NOTHING, so deleting EITHER call + # site kills decoys here and fails the survival loop below. + # + # The override is declared INSIDE the command substitution, which bash runs + # in a subshell, so it cannot outlive this one call — no save/restore to get + # wrong, and no seam in the production path. The kills a broken guard would + # issue are still real, which is what makes the survival loop meaningful. + forced_out="$( + same_selected_process() { return 1; } + reclaim 0 + )" + sleep 1 + if grep -q "pid=${prewarm_pid} was recycled before it could be terminated" <<<"${forced_out}"; then + echo "ok: pre-TERM guard refused to signal a pid whose identity changed" + else + echo "FAIL: pre-TERM guard did not act on a recycled verdict" + failures=$((failures + 1)) + fi + if grep -q "was recycled during the grace period, not escalating" <<<"${forced_out}"; then + echo "ok: pre-KILL guard refused to escalate onto a recycled pid" + else + echo "FAIL: pre-KILL guard did not act on a recycled verdict" + failures=$((failures + 1)) + fi + # The point of both guards: a forced-recycled run must leave every process alive. + for guard_pid in "${decoy_pids[@]}"; do + if ! process_alive "${guard_pid}"; then + echo "FAIL: pid=${guard_pid} was signalled despite a recycled verdict" + failures=$((failures + 1)) + fi + done + # 9. End to end: reclaim kills the leaks and spares all three bystanders — # the manual serve, the harness binary, and the out-of-scope decoy. reclaim_out="$(reclaim 0)" sleep 1 @@ -671,10 +724,10 @@ self_test() { failures=$((failures + 1)) fi - # 9. The escalation ran, and ran only where it was needed. Asserting the - # stubborn decoy died covers the SIGKILL block; asserting the ordinary - # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, - # which would otherwise be silently replaceable by any no-op. + # 10. The escalation ran, and ran only where it was needed. Asserting the + # stubborn decoy died covers the SIGKILL block; asserting the ordinary + # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, + # which would otherwise be silently replaceable by any no-op. if process_alive "${stubborn_pid}"; then echo "FAIL: SIGTERM-ignoring process survived reclaim; escalation to SIGKILL did not happen" failures=$((failures + 1)) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 8deb62dad1..1f5d38e4a3 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2168,9 +2168,15 @@ permissions: .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) .collect(); + // Tracked per WORKFLOW, not per alternation: requiring every + // alternation to name the exempted marker would fail spuriously the + // moment an unrelated `Where-Object … -match` line is added — the + // very case `powershell_reclaim_root_alternations` warns it cannot + // distinguish. One mirror still naming it is what keeps the + // exemption live. + let mut divergence_seen = false; for alternation in &alternations { let present: Vec<&str> = alternation.split('|').collect(); - let mut divergence_seen = false; // Script -> mirror: a root added to the script must reach Windows. for needle in &expected { @@ -2200,19 +2206,19 @@ permissions: or add it there (EAI-8751)" ); } - - // An exemption nothing asserts is an exemption that rots: once - // EAI-8815 moves `__engine-serve-http` out of the root - // alternation, the arm above stops firing and would sit here - // forever as dead code exempting nothing. Fail instead, so the - // fix is told to finish the job. - assert!( - divergence_seen, - "{workflow}'s PowerShell reclaim matcher `{alternation}` no longer names \ - `__engine-serve-http`, so the EAI-8815 divergence looks fixed — delete the \ - exemption arm in this test, which is now dead code (EAI-8751)" - ); } + + // An exemption nothing asserts is an exemption that rots: once + // EAI-8815 moves `__engine-serve-http` out of the root alternation, + // the arm above stops firing and would sit here forever as dead + // code exempting nothing. Fail instead, so the fix is told to + // finish the job. + assert!( + divergence_seen, + "no PowerShell reclaim matcher in {workflow} names `__engine-serve-http`, so the \ + EAI-8815 divergence looks fixed — delete the exemption arm in this test, which \ + is now dead code (EAI-8751)" + ); } } } From ac8303118cdd7444cfb66b1361601bd70a63f042 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 23 Sep 2026 10:00:00 +0000 Subject: [PATCH 09/16] ci(e2e): tighten reclaim self-test evidence and assertions (EAI-8751) Signed-off-by: Tomas Saaristola --- docs/testing.md | 12 ++++++++++++ scripts/reclaim-gpu.sh | 21 +++++++++++++++++++-- xtask/src/workflow_contract.rs | 22 +++++++++++++++++----- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 22d3524d6e..50ac934be3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -835,6 +835,7 @@ cargo test -p rocm-core model_recipe_index_signature_accepts_generated_key_and_r cargo test -p rocm --bin rocm wsl_rocdxg_generated_digest_step_accepts_only_the_matching_file python scripts/release_readiness.py --self-test bash scripts/setup-wsl-portable-build-deps.sh --self-test +bash scripts/reclaim-gpu.sh --self-test ``` The ROCDXG digest check pins the shell fragment the WSL driver plan generates @@ -847,6 +848,17 @@ that is then installed as root, and this replaces the self-test that shipped with the removed `scripts/wsl_setup_rocdxg.sh`. It needs a POSIX shell and `sha256sum`, so it is Unix-only. +`reclaim-gpu.sh` frees the GPU from engine processes a killed prior run leaked. +The self-hosted Linux GPU lanes run it before their GPU preflight; the two +native Windows lanes have no bash and restate the rule in PowerShell instead, +with their root list pinned to the script's by an `xtask` contract test. The +WSL lanes do not run it: they provision a fresh guest per job and unregister it +afterwards, so no prior run's processes survive into them. Its `--self-test` +needs no GPU — it spawns decoy processes and asserts which ones the matching +rule selects — so it runs on the GitHub-hosted lane rather than only where GPU +hardware is held, under the same `heavy` path filter as the other checks there +(which any change under `scripts/` reaches). + The release-readiness self-test is cross-platform and uses only workspace-local temporary files under `.rocm-work/tests/release-readiness`. It also checks exact release asset sets, so stale archives and orphan checksum/signature sidecars in diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 87b9167f07..122c27d44b 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -559,6 +559,11 @@ self_test() { # selection reaching OUTSIDE the scratch tree. An empty selection is the # opposite failure and signals nothing at all, so routing it through that # gate would report a false cause and cut the run short of checks 7-10. + # + # Diagnostic only, and said plainly rather than implied: an empty selection + # is already caught — checks 2 and 5 fail it — so deleting this check does + # NOT let such a run pass. What it adds is the reason, on the run someone + # is reading, which is also why the "ok:" line below is gated on it. if [[ -z "${selected}" ]]; then echo "FAIL: selection was empty; containment had nothing to examine" failures=$((failures + 1)) @@ -591,8 +596,14 @@ self_test() { # Counted on its own rather than off the running total: gating this line on # the total suppressed it whenever an EARLIER check had failed — which is # precisely the run whose output someone is reading. - if [[ "${containment_failures}" -eq 0 ]]; then + # Gated on a non-empty selection as well as on containment: the loop above is + # satisfied trivially by an empty one, so without this the run would print + # "selection was empty" and then immediately claim every selected process was + # checked — evidence contradicting itself on the run being diagnosed. + if [[ "${containment_failures}" -eq 0 && -n "${selected}" ]]; then echo "ok: every selected process lies inside the self-test scratch tree" + elif [[ "${containment_failures}" -eq 0 ]]; then + : # empty selection: already reported above, and nothing to affirm here else # A GATE, not a score. Check 8 below calls the real `reclaim 0`, which # sends real signals to whatever select_leaked returns at that moment. If @@ -680,7 +691,13 @@ self_test() { echo "FAIL: pre-TERM guard did not act on a recycled verdict" failures=$((failures + 1)) fi - if grep -q "was recycled during the grace period, not escalating" <<<"${forced_out}"; then + # Bound to a specific pid, like its pre-TERM sibling above: an unbound match + # would be satisfied by this message emitted for any process. With every + # verdict forced to "recycled" nothing is signalled, so both engine decoys + # survive to the escalation loop and either pid would serve; the stubborn one + # is named because it is the decoy that reaches escalation in the UNforced + # run too, which keeps this assertion reading the same way as check 10. + if grep -q "pid=${stubborn_pid} was recycled during the grace period, not escalating" <<<"${forced_out}"; then echo "ok: pre-KILL guard refused to escalate onto a recycled pid" else echo "FAIL: pre-KILL guard did not act on a recycled verdict" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 1f5d38e4a3..7f33d89efb 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2168,12 +2168,24 @@ permissions: .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) .collect(); - // Tracked per WORKFLOW, not per alternation: requiring every + // Pinned at one, which is what lets the exemption below be tracked + // per WORKFLOW rather than per alternation. Requiring EVERY // alternation to name the exempted marker would fail spuriously the - // moment an unrelated `Where-Object … -match` line is added — the - // very case `powershell_reclaim_root_alternations` warns it cannot - // distinguish. One mirror still naming it is what keeps the - // exemption live. + // moment an unrelated `Where-Object … -match` line appeared — the + // case `powershell_reclaim_root_alternations` warns it cannot + // distinguish — but relaxing it to "some alternation" would stop + // catching a second matcher that copies every root and drops the + // marker. While there is exactly one, the two readings coincide; + // this assertion is what keeps that true, and fails loudly with + // something to decide if a second one is ever added. + assert_eq!( + alternations.len(), + 1, + "{workflow} now has {} PowerShell reclaim matcher alternations; the \ + divergence-exemption check below assumes exactly one, and must be \ + re-read per alternation before this count changes (EAI-8751)", + alternations.len() + ); let mut divergence_seen = false; for alternation in &alternations { let present: Vec<&str> = alternation.split('|').collect(); From f416bf26470ce381a5cf11a886329b8818362e22 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 23 Sep 2026 16:00:48 +0000 Subject: [PATCH 10/16] ci(e2e): reclaim leaks whose argv carries the record delimiters (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit select_leaked emits "pidcmdline" and reclaim reads it back with `IFS=$'\t' read -r`. A tab or newline INSIDE an argv element is content, not a separator, and --chat-template and --prompt carry both. Left intact, one such process emits a record that splits in two: the genuine pid arrives with a truncated command line, fails the identity check against its own full one, and is passed over as "recycled" — so the leak survives TERM and KILL alike while the run signs off with "0 process(es) terminated". That is the silent miss this script exists to end, reproduced against a live decoy before fixing. Flatten TAB and NEWLINE alongside the NUL separators in cmdline_of, the single point every consumer already reads through. Matching is unaffected, since no root or marker contains either byte; the identity comparison does get marginally coarser, which the comment states rather than denies. The PowerShell mirrors pass CIM objects straight to Stop-Process and never serialise a record, so they need no counterpart. report_holders derives its engine filter from ENGINE_MARKERS instead of repeating the alternation, removing the copy that could drift rather than guarding it. -F, not -E, so the markers keep the literal-substring meaning they have in the rule. New self-test check 3 pins the format, with a decoy whose argv carries a real newline and a real tab, asserted to carry both so the check cannot pass having examined a string that could not break it; check 10 adds the end-to-end cost, that such a process is actually reclaimed. Checks 3-10 renumber to 4-11. Signed-off-by: Tomas Saaristola --- scripts/reclaim-gpu.sh | 202 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 183 insertions(+), 19 deletions(-) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 122c27d44b..7499b2140b 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -87,12 +87,33 @@ process_alive() { # Fails when the process is gone or its command line is empty (a kernel thread, # or a zombie whose argv has already been released). # +# TAB and NEWLINE are flattened along with the NUL separators, because they are +# the record delimiters `select_leaked` emits and `reclaim` reads back. Only the +# NUL is a separator the kernel inserted; a tab or newline INSIDE an argv element +# is content, and `--chat-template` or `--prompt` can carry either. Left intact, +# one such process emits a record that splits into two on the way back: the real +# pid arrives carrying a truncated command line, fails the identity check against +# its own full one, and is reported "recycled ... not signalling" — so the leak +# survives both TERM and KILL while the summary prints "0 process(es) terminated". +# That is the silent miss this whole script exists to end, so it is fixed at the +# single point every consumer already reads through rather than at each of them. +# +# Matching is unaffected: no root or marker contains either byte. The identity +# comparison in same_selected_process does get marginally coarser — two command +# lines differing ONLY in which of these bytes sits at a given position now read +# as equal, where before they differed. Both sides come through here, so they +# are flattened alike; the cost is that a recycled pid whose new process differs +# from the old one by nothing but delimiter bytes would be taken for the same +# process. Against the silent miss above, which is reachable by any engine +# invocation carrying a template or a prompt, that is the trade worth making — +# but it is a trade, not a free win. +# # Redirect stderr BEFORE the input redirection: the shell applies them left to # right, so `/dev/null` still lets the shell's own "No such file" reach # the terminal when the open fails. cmdline_of() { local cmdline - cmdline="$(tr '\0' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 + cmdline="$(tr '\0\n\t' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 [[ -n "${cmdline}" ]] || return 1 printf '%s' "${cmdline}" } @@ -257,16 +278,40 @@ reclaim() { # Diagnostics for a preflight that hit its ceiling. Never fails: it runs on the # failure path, where masking the real error would be worse than missing output. report_holders() { + local engine + local -a marker_args=() echo "--- rocm-smi KFD processes (per-process VRAM) ---" # Process names show as UNKNOWN inside a container: KFD reports host PIDs, # which do not resolve in the container's PID namespace. The VRAM column is # still the answer to "what is holding the card". timeout 15 rocm-smi --showpids 2>&1 | head -40 || true echo "--- engine/serve processes visible here ---" + # Derived from ENGINE_MARKERS rather than spelled out again. A second copy of + # the list is exactly the drift the PowerShell mirrors demonstrated and that + # the xtask contract test exists to catch — and this call site had no such + # guard, so a marker added above would have quietly stopped appearing here. + # Deriving it needs no guard: there is no longer a copy that can go stale. + # + # -F, not an -E alternation: the markers are literal substrings everywhere + # else in this script, and building a regex out of them would give a future + # marker containing a metacharacter a different meaning here than in the rule. + # + # No empty-array guard. Given zero patterns grep prints its usage to stderr + # and exits 2; the `|| true` below swallows the STATUS, so the section would + # carry that usage error in place of any holders — wrong, but not silent. + # Reaching it needs ENGINE_MARKERS to be empty, which + # assert_rule_covers_every_list_entry already fails on: it pins the list to + # its four entries by exact comparison, so an emptied array reds the + # self-test before anything gets here. Guarding it again would add a branch + # the self-test could never reach, which is worth less than the assertion + # that already covers it. + for engine in "${ENGINE_MARKERS[@]}"; do + marker_args+=(-e "${engine}") + done # shellcheck disable=SC2009 # pgrep cannot print elapsed time, and how long a # holder has been alive is what distinguishes a leak from this job's own serve. ps -eo pid,etimes,args 2>/dev/null | - grep -Ei 'llama-server|vllm|__engine-serve-http|rocm daemon' | + grep -Fi "${marker_args[@]}" | grep -v grep | head -40 || true echo "--- of those, E2E-owned (reclaim would take these) ---" @@ -344,6 +389,28 @@ DECOY echo $! } +# Spawn a decoy carrying the record delimiters INSIDE one argument — a newline +# and a tab, as a `--chat-template` or `--prompt` value does. +# +# Not spawn_decoy with an extra argument: that decoy is `cp /bin/sleep`, and +# sleep rejects a non-numeric argument and exits before it can be observed. This +# needs something that ignores what it is handed and stays alive. +# +# Ordinary in every other respect — it must die on SIGTERM, so it is the stubborn +# decoy's body without the trap. Its `sleep 1` children name no E2E root, so they +# are never selected, and none outlives the scratch tree by more than a second. +spawn_delimiter_decoy() { + local path="$1" + mkdir -p "$(dirname "${path}")" + cat >"${path}" <<'DECOY' +#!/usr/bin/env bash +while :; do sleep 1; done +DECOY + chmod +x "${path}" + "${path}" $'--chat-template\nrole:\tuser' >/dev/null 2>&1 & + echo $! +} + # Every root and every marker, asserted individually against the rule. # # The expectation is written out rather than derived from the arrays: a loop @@ -403,6 +470,7 @@ self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out local outside outside_decoy outside_pid outside_cmd + local delimiter_decoy delimiter_pid delimiter_raw record_pid local forced_out guard_pid local escapee_pid escapee_cmd guard_rc probe_cmd local zombie_pid zombie_keeper_pid @@ -410,6 +478,8 @@ self_test() { local superseded_hit=0 local probe_failures=0 local containment_failures=0 + local fixture_failures=0 + local record_failures=0 local list_failures=0 local failures=0 @@ -454,21 +524,27 @@ self_test() { stubborn_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" # Matches the rule in full — an E2E root AND an engine marker — but lies # OUTSIDE the scope. It is the only fixture the SELFTEST_SCOPE filter can be - # observed doing anything to, and therefore the only reason check 6 can fail. + # observed doing anything to, and therefore the only reason check 7 can fail. outside_decoy="${outside}/e2e-prewarm-bystander/bin/llama-server" + # Same pre-warm shape again, but its ARGUMENTS carry a newline and a tab — the + # two bytes the pidcmdline record format is built out of. Without + # a fixture whose command line contains them, the format is only ever exercised + # on strings that cannot break it. + delimiter_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" workload_pid="$(spawn_decoy "${workload_decoy}")" harness_pid="$(spawn_decoy "${harness_decoy}")" stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" outside_pid="$(spawn_decoy "${outside_decoy}")" + delimiter_pid="$(spawn_delimiter_decoy "${delimiter_decoy}")" read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" # Every process this function spawned that can still be signalled, so cleanup # is one list rather than a line kept in step at each early return. The zombie # itself is absent deliberately: it is already dead, and killing its keeper is # what lets init reap it. decoy_pids=("${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" - "${outside_pid}" "${zombie_keeper_pid}") + "${outside_pid}" "${delimiter_pid}" "${zombie_keeper_pid}") # Give the decoys a moment to appear in /proc with their full argv. sleep 1 @@ -513,7 +589,86 @@ self_test() { failures=$((failures + 1)) fi - # 3. A manual-testing serve is left alone (engine marker, but no E2E root). + # 3. The record format survives a command line containing the delimiters. + # + # select_leaked emits "pidcmdline" and reclaim reads it back + # with `IFS=$'\t' read -r`. A tab or newline inside an argv element is + # content, not a separator — `--chat-template` and `--prompt` carry both — + # and unless cmdline_of flattens them one record splits into two on the way + # back. The genuine pid then arrives with a TRUNCATED command line, fails + # the identity check against its own full one, and is passed over as + # "recycled": the leak survives TERM and KILL alike while the run signs off + # with "0 process(es) terminated". A silent miss is the one outcome this + # script exists to prevent, so the format is asserted rather than assumed. + # + # Placed HERE, ahead of containment, deliberately. A split record also + # trips check 7's escape loop — its continuation line names no scratch + # tree — which arms the gate and returns before checks 8-11 ever run. The + # run is red either way; what this adds is the true cause, printed before + # the one check 7 would otherwise report in its place. + # Read RAW, flattening only the kernel's NUL separators: cmdline_of now + # flattens the delimiters too, so asking it would report the fixture is fine + # no matter what the decoy was actually given. Both bytes are asserted + # individually, because either one alone breaks the format differently — the + # newline splits the record, the tab shifts the field boundary. + # Counted apart from the record loop below, for the same reason check 7 keeps + # its empty-selection count separate: these say the FIXTURE is unusable, the + # loop says the FORMAT is broken, and the "ok:" line must not be able to + # affirm the second while the first has just been denied. + # stderr redirected BEFORE the input, for the reason cmdline_of spells out: + # the shell applies them left to right, so the input-first form lets its own + # "No such file" reach the terminal on a failed open — here, noise printed by + # the very branch that exists to report a dead fixture cleanly. + if ! delimiter_raw="$(tr '\0' ' ' 2>/dev/null <"/proc/${delimiter_pid}/cmdline")"; then + # Distinguished from "carries no delimiter": if the decoy died before this + # read, the byte assertions below would report a format problem for what is + # really a dead fixture, and send the next reader after the wrong thing. + echo "FAIL: delimiter decoy's command line could not be read; the fixture is gone, not malformed" + fixture_failures=$((fixture_failures + 1)) + delimiter_raw='' + else + case "${delimiter_raw}" in + *$'\n'*) ;; + *) + echo "FAIL: delimiter decoy's command line carries no newline; the record-format check is vacuous" + fixture_failures=$((fixture_failures + 1)) + ;; + esac + case "${delimiter_raw}" in + *$'\t'*) ;; + *) + echo "FAIL: delimiter decoy's command line carries no tab; the record-format check is vacuous" + fixture_failures=$((fixture_failures + 1)) + ;; + esac + fi + if ! grep -q "^${delimiter_pid} " <<<"${selected}"; then + # Selection is NOT what this check tests — and note it survives the + # unflattened form, because the `^pid` grep matches the first line of a + # split record. It is asserted only so the loop below cannot report a pass + # having never seen a command line with a delimiter in it. + echo "FAIL: delimiter decoy was not selected; the record-format check has no fixture" + fixture_failures=$((fixture_failures + 1)) + fi + failures=$((failures + fixture_failures)) + while IFS=$'\t' read -r record_pid _; do + [[ -n "${record_pid}" ]] || continue + if [[ ! "${record_pid}" =~ ^[0-9]+$ ]]; then + echo "FAIL: selection record does not begin with a pid: '${record_pid}'" + record_failures=$((record_failures + 1)) + fi + done <<<"${selected}" + failures=$((failures + record_failures)) + # Gated on the fixture too, not just on the record loop. A loop that examined + # a command line with no delimiter in it proves nothing, so affirming it here + # would print "every record survives delimiters" directly beneath a FAIL line + # saying there were none — evidence contradicting itself on the run someone is + # reading, which is the very thing check 7 is shaped to avoid. + if [[ "${record_failures}" -eq 0 && "${fixture_failures}" -eq 0 ]]; then + echo "ok: every selection record survives delimiters in a command line" + fi + + # 4. A manual-testing serve is left alone (engine marker, but no E2E root). if grep -q "^${workload_pid} " <<<"${selected}"; then echo "FAIL: /workload manual serve was selected; reclaim must not touch it" failures=$((failures + 1)) @@ -521,7 +676,7 @@ self_test() { echo "ok: /workload manual serve is not selected" fi - # 4. Both halves are required: an E2E root alone must not select. Deleting the + # 5. Both halves are required: an E2E root alone must not select. Deleting the # `has_engine` requirement makes exactly this check fail and nothing else. if grep -q "^${harness_pid} " <<<"${selected}"; then echo "FAIL: E2E test binary was selected; the engine half of the rule is not enforced" @@ -530,7 +685,7 @@ self_test() { echo "ok: an E2E root without an engine marker is not selected" fi - # 5. A process that ignores SIGTERM is still selected. + # 6. A process that ignores SIGTERM is still selected. if grep -q "^${stubborn_pid} " <<<"${selected}"; then echo "ok: SIGTERM-ignoring pre-warm engine process is selected" else @@ -538,7 +693,7 @@ self_test() { failures=$((failures + 1)) fi - # 6. Containment: self_test issues REAL kills, and SELFTEST_SCOPE is the only + # 7. Containment: self_test issues REAL kills, and SELFTEST_SCOPE is the only # thing keeping them inside the scratch tree. Assert that before killing # rather than trusting it — this step runs on a hosted ephemeral lane # today, but nothing in the script stops it being run anywhere else. @@ -551,17 +706,17 @@ self_test() { # thing the filter can be caught NOT doing its job on. # An empty selection satisfies the loop below trivially, so without this the # "ok:" line would report containment verified on a run where nothing was - # examined. Checks 2 and 5 already fail such a run, so this is not a false + # examined. Checks 2 and 6 already fail such a run, so this is not a false # green — but the evidence line is read by whoever is diagnosing that run. # # Counted apart from containment_failures on purpose: that counter arms the # gate below, whose message and early return are specifically about a # selection reaching OUTSIDE the scratch tree. An empty selection is the # opposite failure and signals nothing at all, so routing it through that - # gate would report a false cause and cut the run short of checks 7-10. + # gate would report a false cause and cut the run short of checks 8-11. # # Diagnostic only, and said plainly rather than implied: an empty selection - # is already caught — checks 2 and 5 fail it — so deleting this check does + # is already caught — checks 2 and 6 fail it — so deleting this check does # NOT let such a run pass. What it adds is the reason, on the run someone # is reading, which is also why the "ok:" line below is gated on it. if [[ -z "${selected}" ]]; then @@ -605,7 +760,7 @@ self_test() { elif [[ "${containment_failures}" -eq 0 ]]; then : # empty selection: already reported above, and nothing to affirm here else - # A GATE, not a score. Check 8 below calls the real `reclaim 0`, which + # A GATE, not a score. Check 9 below calls the real `reclaim 0`, which # sends real signals to whatever select_leaked returns at that moment. If # containment has just failed, that selection reaches outside this scratch # tree — the exact accident this check exists to prevent — so scoring it @@ -617,10 +772,10 @@ self_test() { return 1 fi - # 7. The escalation guard's comparison, in all three directions — and for + # 8. The escalation guard's comparison, in all three directions — and for # "gone", by BOTH routes into it. # - # The COMPARISON is covered here; check 8 covers the two CALL SITES by + # The COMPARISON is covered here; check 9 covers the two CALL SITES by # forcing the verdict, because a pid cannot be made to be reused by a # different process on demand. if same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")"; then @@ -667,7 +822,7 @@ self_test() { failures=$((failures + 1)) fi - # 8. The guard's two CALL SITES, by forcing the verdict they act on. + # 9. The guard's two CALL SITES, by forcing the verdict they act on. # # A pid cannot be made to be reused by a different process on demand, so # until now both call sites were exercised only in the always-proceed @@ -696,7 +851,7 @@ self_test() { # verdict forced to "recycled" nothing is signalled, so both engine decoys # survive to the escalation loop and either pid would serve; the stubborn one # is named because it is the decoy that reaches escalation in the UNforced - # run too, which keeps this assertion reading the same way as check 10. + # run too, which keeps this assertion reading the same way as check 11. if grep -q "pid=${stubborn_pid} was recycled during the grace period, not escalating" <<<"${forced_out}"; then echo "ok: pre-KILL guard refused to escalate onto a recycled pid" else @@ -710,7 +865,7 @@ self_test() { failures=$((failures + 1)) fi done - # 9. End to end: reclaim kills the leaks and spares all three bystanders — + # 10. End to end: reclaim kills the leaks and spares all three bystanders — # the manual serve, the harness binary, and the out-of-scope decoy. reclaim_out="$(reclaim 0)" sleep 1 @@ -732,8 +887,17 @@ self_test() { echo "FAIL: E2E test binary was killed by reclaim" failures=$((failures + 1)) fi + # The end-to-end form of check 3, and the one that states the cost: with the + # delimiters unflattened this process is selected, passed over as "recycled", + # and still holding the card when reclaim reports success. + if process_alive "${delimiter_pid}"; then + echo "FAIL: engine process with delimiters in its command line survived reclaim" + failures=$((failures + 1)) + else + echo "ok: engine process with delimiters in its command line was reclaimed" + fi # The bystander matches the rule in full, so only the scope kept it out of the - # selection. Its survival is the end-to-end form of check 6. + # selection. Its survival is the end-to-end form of check 7. if process_alive "${outside_pid}"; then echo "ok: rule-matching process outside the scratch tree survived reclaim" else @@ -741,7 +905,7 @@ self_test() { failures=$((failures + 1)) fi - # 10. The escalation ran, and ran only where it was needed. Asserting the + # 11. The escalation ran, and ran only where it was needed. Asserting the # stubborn decoy died covers the SIGKILL block; asserting the ordinary # decoy did NOT reach escalation covers the `kill -TERM` that precedes it, # which would otherwise be silently replaceable by any no-op. From 35cbc77b9d9d514244785fe5babd62d3ab6a4d7f Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 25 Sep 2026 09:30:04 +0000 Subject: [PATCH 11/16] ci(e2e): anchor the scenario root and stop overclaiming the rest (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two blocking findings from review, plus the two regressions fixing them introduced. The tab half of cmdline_of's flattening was inert. Deleting only the tab left every check in this file green, and the mechanism the comment claimed does not occur: `read` gives leftover separators to the LAST name, and flattening the final NUL always leaves a trailing space, so an embedded tab is never stripped as a trailing separator and round-trips verbatim. Only a newline splits a record. The tab is no longer flattened, which also gives back the identity precision that flattening it cost for nothing. Its fixture stays, re-framed as what it actually guards: the record FORMAT, not a flattening step. The scenario root now follows TMPDIR. Scenario dirs come from `std::env::temp_dir()`, and the nightly Strix lane redirects it out of /tmp; the absolute `/tmp/rocm-e2e` kept matching there only because that directory is itself named `tmp`, and would lose any redirect not so named. The redirected form is derived at load time, which covers ANY target. The cleanup glob gets the same treatment — it removed nothing at all on that lane, so the disk-fill guard was dead exactly where it was needed. Deriving it rather than using the bare `rocm-e2e` segment is deliberate, and was learned the hard way: roots are matched as unanchored substrings of the whole command line, so the bare form also selects a process that merely NAMES one in an argument. `--hf-repo myorg/rocm-e2e-baseline-7b` on a hand-run serve was selected and would have been SIGKILLed — the one thing the /workload rule promises to spare. Two assertions now pin that, since the existing /workload fixtures carry no arguments and stayed green against it. Anchoring narrows that surface without closing it, and the header now says so instead of claiming otherwise: an argument naming a sibling of a redirected TMPDIR still matches, and the three segment roots are not anchored at all, so `--model-path /home/dev/e2e-shared-models/x.gguf` is selected today. Both predate this script and neither is fixed here. The xtask comment no longer files that gap under EAI-8815 either, which covers only the engine-marker divergence. A relative TMPDIR is now ignored at all four consumers. It matters most at the `rm -rf`, the only one that deletes rather than failing to match: interpolated raw it resolves against the CWD, which on a runner is the checkout. ci.yml runs the self-test twice, once with TMPDIR redirected, because a hosted runner has none and the redirect arm would otherwise never execute there. Signed-off-by: Tomas Saaristola --- .github/workflows/ci.yml | 11 +- scripts/reclaim-gpu.sh | 195 ++++++++++++++++++++++++++------- xtask/src/workflow_contract.rs | 24 +++- 3 files changed, 190 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d22b24e63..82f13f317d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -382,7 +382,16 @@ jobs: # provable against decoy processes with no GPU present. - name: GPU reclaim self-test if: needs.changes.outputs.heavy == 'true' - run: bash scripts/reclaim-gpu.sh --self-test + # Twice, because one of the rules is conditional on the environment. + # The scenario root follows TMPDIR, and the self-test can only assert + # the redirected form when the shell running it actually has one — a + # hosted runner does not, so a single plain run reports that arm as + # "not exercised" and the nightly Strix lane's redirect would be the + # first place a break showed up. The second run supplies the redirect. + run: | + bash scripts/reclaim-gpu.sh --self-test + mkdir -p "${RUNNER_TEMP}/redirected-tmp" + TMPDIR="${RUNNER_TEMP}/redirected-tmp" bash scripts/reclaim-gpu.sh --self-test - name: Release readiness self-test if: needs.changes.outputs.heavy == 'true' diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 7499b2140b..01866ab1e3 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -30,17 +30,59 @@ set -euo pipefail # E2E-owned roots. A process must name one of these to be considered ours. -# /tmp/rocm-e2e per-scenario temp dirs +# /tmp/rocm-e2e per-scenario temp dirs (plus the TMPDIR-derived form below) # e2e-shared E2E_SHARED_CACHE_DIR (models, HF weights) # e2e-prewarm E2E_SHARED_RUNTIMES_DIR — the pre-warmed runtime tree the # engine binaries actually live in # e2e-target CARGO_TARGET_DIR for the suite's own binaries +# +# The scenario root stays ABSOLUTE, and follows TMPDIR. +# +# Scenario dirs come from `TempDir::with_prefix("rocm-e2e-")`, which builds on +# `std::env::temp_dir()` and so follows TMPDIR; the nightly Strix lane redirects +# it to `$HOME/actions-runner/tmp`. A bare `/tmp/rocm-e2e` kept matching there +# only by accident — that directory is itself named `tmp`, so the substring +# still appeared — and would silently stop matching under any redirect not +# named `tmp`. Deriving the redirected form fixes that for ANY target. +# +# WHAT THE ANCHORING DOES AND DOES NOT BUY. Roots are matched as unanchored +# substrings of the WHOLE command line, so any root also matches a process that +# merely mentions it in an ARGUMENT. The bare segment `rocm-e2e` — what both +# PowerShell mirrors use — makes that trivially reachable: a hand-run +# `--hf-repo myorg/rocm-e2e-baseline-7b` on a shared runner is selected and +# SIGKILLed. Measured, and briefly shipped, which is why it is spelled out. +# +# Anchoring the scenario root to an absolute path NARROWS that surface. It does +# not close it, and the `/workload` promise below is correspondingly weaker than +# it reads: +# +# - under a redirected TMPDIR, an argument naming a sibling path still +# matches — `--extra-data-dir /tmp/mydir/rocm-e2e-notes` with +# TMPDIR=/tmp/mydir; +# - the three segment roots are NOT anchored at all, and "shared" and "target" +# are ordinary words. `--model-path /home/dev/e2e-shared-models/x.gguf` and +# `--served-model-name e2e-target-vs-baseline` are both selected today. +# +# Both predate this script and neither is fixed here; closing them properly +# means matching on a path boundary rather than a substring, across the bash +# rule and both mirrors together. Stated rather than implied, because an earlier +# revision of this comment claimed these roots "name directories no argument +# plausibly carries" — which is false, and is the same overclaim that made the +# bare-segment change look safe. E2E_ROOTS=( '/tmp/rocm-e2e' 'e2e-shared' 'e2e-prewarm' 'e2e-target' ) +# Appended rather than listed, because it is only known at run time. Skipped +# when TMPDIR is unset or already /tmp, so the list stays exactly the pinned one +# on every lane that does not redirect. A RELATIVE TMPDIR is skipped too: it +# would append a relative root and quietly give up the anchoring above, and no +# lane sets one. +if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + E2E_ROOTS+=("${TMPDIR%/}/rocm-e2e") +fi # Engine/serve processes that can hold VRAM. Matched anywhere in the command # line, so a wrapper or an absolute binary path both work. @@ -87,33 +129,45 @@ process_alive() { # Fails when the process is gone or its command line is empty (a kernel thread, # or a zombie whose argv has already been released). # -# TAB and NEWLINE are flattened along with the NUL separators, because they are -# the record delimiters `select_leaked` emits and `reclaim` reads back. Only the -# NUL is a separator the kernel inserted; a tab or newline INSIDE an argv element -# is content, and `--chat-template` or `--prompt` can carry either. Left intact, -# one such process emits a record that splits into two on the way back: the real -# pid arrives carrying a truncated command line, fails the identity check against -# its own full one, and is reported "recycled ... not signalling" — so the leak -# survives both TERM and KILL while the summary prints "0 process(es) terminated". -# That is the silent miss this whole script exists to end, so it is fixed at the -# single point every consumer already reads through rather than at each of them. +# NEWLINE is flattened along with the NUL separators the kernel inserted, +# because it is the record terminator `select_leaked` emits and `reclaim` reads +# back. A newline INSIDE an argv element is content, not a separator, and +# `--chat-template` or `--prompt` can carry one. Left intact, such a process +# emits a record that splits into two on the way back: the real pid arrives +# carrying a truncated command line, fails the identity check against its own +# full one, and is reported "recycled ... not signalling" — so the leak survives +# both TERM and KILL while the summary prints "0 process(es) terminated". That is +# the silent miss this whole script exists to end, so it is fixed at the single +# point every consumer already reads through rather than at each of them. # -# Matching is unaffected: no root or marker contains either byte. The identity -# comparison in same_selected_process does get marginally coarser — two command -# lines differing ONLY in which of these bytes sits at a given position now read -# as equal, where before they differed. Both sides come through here, so they -# are flattened alike; the cost is that a recycled pid whose new process differs -# from the old one by nothing but delimiter bytes would be taken for the same -# process. Against the silent miss above, which is reachable by any engine -# invocation carrying a template or a prompt, that is the trade worth making — -# but it is a trade, not a free win. +# The TAB is deliberately NOT flattened, though it is the field separator. It +# does not need to be, measured rather than reasoned: +# +# record "\t/path/llama-server --tmpl a\tb " +# read -r pid cmdline -> pid=[] cmdline=[/path/llama-server --tmpl a\tb ] +# +# `read` gives leftover words AND their intervening separators to the LAST name, +# so an embedded tab lands in `cmdline` verbatim; and only TRAILING separators +# are stripped, which a tab can never be, because flattening the final NUL always +# leaves the string ending in a space. An earlier revision of this comment +# claimed the tab "shifts the field boundary" and flattened it too — that claim +# was false, and deleting the tab from the flattening set failed no check in this +# file. Flattening it anyway would have coarsened the identity comparison below +# for nothing. +# +# Matching is unaffected either way: no root or marker contains a newline. The +# identity comparison in same_selected_process is coarsened only for the newline +# — two command lines differing ONLY in a newline-versus-space at one position +# now read as equal. Both sides come through here, so they are flattened alike. +# That is a real cost, weighed against a silent miss any engine invocation +# carrying a prompt can reach. # # Redirect stderr BEFORE the input redirection: the shell applies them left to # right, so `/dev/null` still lets the shell's own "No such file" reach # the terminal when the open fails. cmdline_of() { local cmdline - cmdline="$(tr '\0\n\t' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 + cmdline="$(tr '\0\n' ' ' 2>/dev/null <"/proc/${1}/cmdline")" || return 1 [[ -n "${cmdline}" ]] || return 1 printf '%s' "${cmdline}" } @@ -424,6 +478,13 @@ assert_rule_covers_every_list_entry() { local root engine local expected_roots=('/tmp/rocm-e2e' 'e2e-shared' 'e2e-prewarm' 'e2e-target') local expected_markers=('llama-server' 'vllm' '__engine-serve-http' 'rocm daemon') + # The TMPDIR-derived scenario root is appended at load time, so the pinned + # list is the STATIC one plus that entry when it applies. Computed the same + # way the script does rather than assumed absent: this function also runs on a + # developer machine, and macOS sets TMPDIR for every shell. + if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + expected_roots+=("${TMPDIR%/}/rocm-e2e") + fi if [[ "$(printf '%s\n' "${E2E_ROOTS[@]}")" != "$(printf '%s\n' "${expected_roots[@]}")" ]]; then echo "FAIL: E2E_ROOTS changed — update this expectation AND both PowerShell mirrors" @@ -460,6 +521,39 @@ assert_rule_covers_every_list_entry() { fi done + # The scenario root must stay ANCHORED to an absolute path. These are argument + # strings, not paths: a hand-run serve on a shared runner that merely NAMES an + # E2E-suite baseline. A bare `rocm-e2e` root matches both and SIGKILLs them. + # The `/workload` fixtures above cannot catch it — they carry no arguments at + # all, so they stay green against exactly this mistake. It was made, and this + # is what would have caught it. + # + # This pins the scenario root ONLY. The header explains why that is a + # narrowing rather than a fix: the three segment roots stay unanchored, so + # `--model-path /home/dev/e2e-shared-models/x.gguf` is still selected. No + # assertion here claims otherwise, deliberately — a green run means the + # scenario root did not regress, not that a manual serve is safe. + for root in '--hf-repo myorg/rocm-e2e-baseline-7b' '--model-alias my-rocm-e2e-comparison'; do + if cmdline_matches_rule "/workload/manual/llama-server ${root}"; then + echo "FAIL: a manual serve was selected for merely NAMING '${root}'; the scenario root is not anchored" + failures=$((failures + 1)) + fi + done + + # The other half of the same decision: anchoring must not cost the redirect. + # Only assertable when this shell actually has a redirected TMPDIR, so it says + # which case ran rather than reporting a pass for a branch it skipped. + if [[ "${TMPDIR:-}" == /* && "${TMPDIR%/}" != "/tmp" ]]; then + if cmdline_matches_rule "${TMPDIR%/}/rocm-e2e-ab12/bin/llama-server --model m"; then + echo "ok: a scenario tree under the redirected TMPDIR is selected" + else + echo "FAIL: TMPDIR is redirected but a scenario tree under it is not selected" + failures=$((failures + 1)) + fi + else + echo "note: no absolute TMPDIR redirect in this shell, so that arm was not exercised" + fi + if [[ "${failures}" -eq 0 ]]; then echo "ok: every E2E root and every engine marker is individually enforced" fi @@ -589,28 +683,38 @@ self_test() { failures=$((failures + 1)) fi - # 3. The record format survives a command line containing the delimiters. + # 3. The record format survives a command line containing BOTH delimiter bytes. # # select_leaked emits "pidcmdline" and reclaim reads it back - # with `IFS=$'\t' read -r`. A tab or newline inside an argv element is - # content, not a separator — `--chat-template` and `--prompt` carry both — - # and unless cmdline_of flattens them one record splits into two on the way - # back. The genuine pid then arrives with a TRUNCATED command line, fails - # the identity check against its own full one, and is passed over as - # "recycled": the leak survives TERM and KILL alike while the run signs off - # with "0 process(es) terminated". A silent miss is the one outcome this - # script exists to prevent, so the format is asserted rather than assumed. + # with `IFS=$'\t' read -r`. The two bytes are not symmetric, and this check + # is the only place that says so: + # + # - a NEWLINE in an argv element splits one record into two. The genuine pid + # then arrives with a TRUNCATED command line, fails the identity check + # against its own full one, and is passed over as "recycled": the leak + # survives TERM and KILL alike while the run signs off with "0 process(es) + # terminated". cmdline_of flattens it for that reason, and deleting that + # flattening fails this check. + # - a TAB does NOT, even though it is the field separator, because `read` + # hands leftover separators to the last name and the trailing space left + # by the final NUL keeps a tab from ever being stripped as a trailing one. + # cmdline_of deliberately leaves it alone; see the reasoning there. + # + # The tab fixture therefore guards the FORMAT, not the flattening: it is + # what fails if the record shape is ever changed to something a tab can + # break. Asserting it as though it pinned a flattening step would be a claim + # of coverage that no mutation can falsify, which is how this check read + # before — it announced the tab as individually asserted while deleting the + # tab from the flattening set left every check green. # # Placed HERE, ahead of containment, deliberately. A split record also # trips check 7's escape loop — its continuation line names no scratch # tree — which arms the gate and returns before checks 8-11 ever run. The # run is red either way; what this adds is the true cause, printed before # the one check 7 would otherwise report in its place. - # Read RAW, flattening only the kernel's NUL separators: cmdline_of now - # flattens the delimiters too, so asking it would report the fixture is fine - # no matter what the decoy was actually given. Both bytes are asserted - # individually, because either one alone breaks the format differently — the - # newline splits the record, the tab shifts the field boundary. + # Read RAW, flattening only the kernel's NUL separators: cmdline_of flattens + # the newline, so asking it would report the fixture carries one no matter what + # the decoy was actually given. # Counted apart from the record loop below, for the same reason check 7 keeps # its empty-selection count separate: these say the FIXTURE is unusable, the # loop says the FORMAT is broken, and the "ok:" line must not be able to @@ -637,7 +741,7 @@ self_test() { case "${delimiter_raw}" in *$'\t'*) ;; *) - echo "FAIL: delimiter decoy's command line carries no tab; the record-format check is vacuous" + echo "FAIL: delimiter decoy's command line carries no tab; the record format is no longer exercised against one" fixture_failures=$((fixture_failures + 1)) ;; esac @@ -942,12 +1046,29 @@ self_test() { } main() { + local scenario_tmp case "${1:-}" in '') reclaim 0 # Scenario temp dirs are recreated per run; clearing them keeps a wedged # runner's disk from filling with dead scenario state. - rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + # + # TMPDIR as well as /tmp, because the suite creates these through + # `std::env::temp_dir()` and the nightly Strix lane redirects TMPDIR to + # the runner's home. A bare `/tmp` glob removed nothing at all there, so + # the guard was dead on the one lane that redirects. Both are listed + # rather than just TMPDIR: the lanes that do not set it still want /tmp, + # and a stale tree from before a redirect was added would outlive it. + # + # The absolute-path test is the SAME one the roots above apply, and it + # matters most here: this is the only consumer that deletes rather than + # merely failing to match. Interpolated raw, a relative TMPDIR makes this + # `rm -rf relative/dir/rocm-e2e-*` resolved against the CWD — the repo + # checkout, on a CI runner. No lane sets one, which is a reason to skip + # the value, not a reason to hand it to `rm -rf` unchecked. + scenario_tmp="${TMPDIR:-}" + [[ "${scenario_tmp}" == /* ]] || scenario_tmp='/tmp' + rm -rf "${scenario_tmp%/}"/rocm-e2e-* /tmp/rocm-e2e-* 2>/dev/null || true ;; --dry-run) reclaim 1 ;; --report-holders) report_holders ;; diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 7f33d89efb..483f974f57 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2161,8 +2161,28 @@ permissions: "{workflow} has a PowerShell reclaim but no parsable `-match` alternation \ — the step's shape changed and this guard went blind (EAI-8751)" ); - // `/tmp/rocm-e2e` is POSIX-only; the mirrors match the `rocm-e2e` - // segment, the portable part of the same root. + // The scenario root is compared on its portable segment, and that + // asymmetry is deliberate rather than cosmetic. + // + // bash anchors it to an absolute path (`/tmp/rocm-e2e`, plus a + // TMPDIR-derived form appended at run time) because its roots are + // matched as unanchored substrings of a whole command line: the + // bare segment would also match an ARGUMENT naming it, and kill a + // hand-run serve the script promises to spare. The PowerShell + // mirrors carry the bare segment and so do have that exposure — + // pre-existing, and not something this test can fix by failing. + // + // That anchoring gap is NOT what EAI-8815 tracks. That ticket is + // scoped to the engine-marker divergence (`rocm.exe daemon` + // unmatched; `__engine-serve-http` sitting in the root alternation + // rather than the engine one), and closing it would leave the + // substring exposure untouched. The anchoring gap is untracked on + // the Windows side — said plainly here, so that closing EAI-8815 + // cannot be misread as closing this as well. + // + // What it still pins is the part that matters here: that every root + // the script knows about is named in both mirrors, so a root added + // to one side cannot silently miss the Windows lanes. let expected: Vec<&str> = roots .iter() .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) From 8e871c2c34f2c77aa8f60883c63e0670878f02f6 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 25 Sep 2026 09:58:26 +0000 Subject: [PATCH 12/16] ci(e2e): stop the self-test leaking decoys, and correct two claims (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing findings from a full-scope pass. None blocking; two are false sentences I wrote, which is the third round in a row that has been the defect. self_test registered its decoys with the EXIT trap only after all six spawns had succeeded, but every spawn is fallible (mkdir, cp, chmod) and the function runs under `set -e`. A failure partway left the already-spawned decoys running with nothing arranged to kill them — including the stubborn one, which ignores SIGTERM and loops forever. Measured against an injected mid-spawn failure: three survivors before, none after. Each pid is now tracked the moment it exists, the same way the scratch-directory traps are widened one statement at a time. cmdline_of's comment claimed separators are stripped only when TRAILING, "which a tab can never be". Leading separators of the trailing field are stripped too — `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`. Reaching it needs an argv[0] that itself begins with a tab, which no caller produces, so the conclusion stands and the reasoning did not. docs/testing.md said the WSL lanes "do not run it". They skip the proactive reclaim before preflight, which is what the fresh-guest-per-job argument supports, but they do call the script for --report-holders on preflight failure like every other lane. Left alone deliberately: the WSL lane's --report-holders call does not cd into the checkout first, unlike its two sibling WSL steps, so it may not resolve. Pre-existing, and not verifiable without a WSL guest. Signed-off-by: Tomas Saaristola --- docs/testing.md | 6 +++-- scripts/reclaim-gpu.sh | 61 +++++++++++++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 50ac934be3..fa8a41e21e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -852,8 +852,10 @@ with the removed `scripts/wsl_setup_rocdxg.sh`. It needs a POSIX shell and The self-hosted Linux GPU lanes run it before their GPU preflight; the two native Windows lanes have no bash and restate the rule in PowerShell instead, with their root list pinned to the script's by an `xtask` contract test. The -WSL lanes do not run it: they provision a fresh guest per job and unregister it -afterwards, so no prior run's processes survive into them. Its `--self-test` +WSL lanes skip the proactive reclaim before preflight — they provision a fresh +guest per job and unregister it afterwards, so no prior run's processes survive +into them — but they still call the script for `--report-holders` when their +preflight fails, like every other lane. Its `--self-test` needs no GPU — it spawns decoy processes and asserts which ones the matching rule selects — so it runs on the GitHub-hosted lane rather than only where GPU hardware is held, under the same `heavy` path filter as the other checks there diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 01866ab1e3..5433c81107 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -147,13 +147,20 @@ process_alive() { # read -r pid cmdline -> pid=[] cmdline=[/path/llama-server --tmpl a\tb ] # # `read` gives leftover words AND their intervening separators to the LAST name, -# so an embedded tab lands in `cmdline` verbatim; and only TRAILING separators -# are stripped, which a tab can never be, because flattening the final NUL always -# leaves the string ending in a space. An earlier revision of this comment -# claimed the tab "shifts the field boundary" and flattened it too — that claim -# was false, and deleting the tab from the flattening set failed no check in this -# file. Flattening it anyway would have coarsened the identity comparison below -# for nothing. +# so a tab EMBEDDED in the command line lands in `cmdline` verbatim. Separators +# are stripped only where the assigned field BEGINS or ENDS with them, and the +# command line is argv[0] first — a path — so neither applies here. +# +# That is narrower than it first read. Leading separators of the trailing field +# are stripped too: `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`, not +# a tab then `x`. It would take an argv[0] that itself began with a tab to reach +# that, which no caller of this script produces — but "a tab can never be +# stripped" is not the true statement, and this comment said so for a revision. +# +# An earlier revision also claimed the tab "shifts the field boundary" and +# flattened it for that reason — also false, and deleting the tab from the +# flattening set failed no check in this file. Flattening it anyway would have +# coarsened the identity comparison below for nothing. # # Matching is unaffected either way: no root or marker contains a newline. The # identity comparison in same_selected_process is coarsened only for the newline @@ -560,6 +567,24 @@ assert_rule_covers_every_list_entry() { return "${failures}" } +# Decoys the self-test has spawned so far, read by its EXIT trap. +# +# Global, and read at trap FIRE time rather than expanded at definition time the +# way the scratch-dir paths are: a pid is not known until its spawn returns, so +# the one thing that must not be baked into the trap string is this list. +SELFTEST_DECOY_PIDS=() + +selftest_track_decoy() { + SELFTEST_DECOY_PIDS+=("$1") +} + +# SIGKILL, not TERM: one decoy ignores TERM by design, and this runs on the way +# out with nothing left to wait for it. +selftest_kill_decoys() { + [[ "${#SELFTEST_DECOY_PIDS[@]}" -gt 0 ]] || return 0 + kill -KILL "${SELFTEST_DECOY_PIDS[@]}" 2>/dev/null || true +} + self_test() { local tmp prewarm_decoy workload_decoy harness_decoy stubborn_decoy local prewarm_pid workload_pid harness_pid stubborn_pid selected reclaim_out @@ -588,7 +613,7 @@ self_test() { # that aborts the function — with `tmp` already on disk and, if the trap were # installed only afterwards, nothing left to clean it up. # shellcheck disable=SC2064 # expand ${tmp} now, at trap definition time - trap "rm -rf '${tmp}'" EXIT + trap "selftest_kill_decoys; rm -rf '${tmp}'" EXIT # A SECOND tree, outside the scope, for the bystander decoy below. Everything # reachable through SELFTEST_SCOPE is inside `tmp` by construction, so a # negative case for the scope filter cannot live there. @@ -597,7 +622,7 @@ self_test() { # same reason the narrow trap above exists: anything fallible in between is a # window where a tree is on disk with nothing arranged to remove it. # shellcheck disable=SC2064 # expand both paths now, at trap definition time - trap "rm -rf '${tmp}' '${outside}'" EXIT + trap "selftest_kill_decoys; rm -rf '${tmp}' '${outside}'" EXIT export RECLAIM_SELFTEST_SCOPE="${tmp}" SELFTEST_SCOPE="${tmp}" # The stubborn decoy never exits on its own, so the grace loop always runs to @@ -626,19 +651,31 @@ self_test() { # on strings that cannot break it. delimiter_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" + # Each pid is registered with the EXIT trap the moment it exists, for the same + # reason the scratch-dir traps above are widened one statement at a time: every + # spawn below is fallible (mkdir, cp, chmod), and `set -e` aborts the function + # on the first failure. Registering them only in the list further down would + # leave every ALREADY-spawned decoy running with nothing arranged to kill it — + # including the stubborn one, which ignores SIGTERM and loops forever. prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" + selftest_track_decoy "${prewarm_pid}" workload_pid="$(spawn_decoy "${workload_decoy}")" + selftest_track_decoy "${workload_pid}" harness_pid="$(spawn_decoy "${harness_decoy}")" + selftest_track_decoy "${harness_pid}" stubborn_pid="$(spawn_stubborn_decoy "${stubborn_decoy}")" + selftest_track_decoy "${stubborn_pid}" outside_pid="$(spawn_decoy "${outside_decoy}")" + selftest_track_decoy "${outside_pid}" delimiter_pid="$(spawn_delimiter_decoy "${delimiter_decoy}")" + selftest_track_decoy "${delimiter_pid}" read -r zombie_pid zombie_keeper_pid <<<"$(spawn_zombie "${tmp}/zombie")" + selftest_track_decoy "${zombie_keeper_pid}" # Every process this function spawned that can still be signalled, so cleanup # is one list rather than a line kept in step at each early return. The zombie # itself is absent deliberately: it is already dead, and killing its keeper is - # what lets init reap it. - decoy_pids=("${prewarm_pid}" "${workload_pid}" "${harness_pid}" "${stubborn_pid}" - "${outside_pid}" "${delimiter_pid}" "${zombie_keeper_pid}") + # what lets init reap it. Same set the trap holds, by construction. + decoy_pids=("${SELFTEST_DECOY_PIDS[@]}") # Give the decoys a moment to appear in /proc with their full argv. sleep 1 From b56fb8bc2dc9b6f110a6e8fe1c133c6f117d81ba Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 25 Sep 2026 11:17:38 +0000 Subject: [PATCH 13/16] ci(e2e): correct three comment claims the code does not make (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three blocking findings from review are sentences, not behaviour. Verified each by measurement before changing it. The spawn block justified early registration with "`set -e` aborts the function on the first failure". It does not: `inherit_errexit` is off, so errexit never reaches inside `$( )`, and every spawn helper ends in an `echo` that succeeds — a hard failure injected into a helper runs the suite to a green exit 0. The guard is still right, for the routes that do reach the trap mid-way: a signal, and any fallible statement between the spawns. The consequence of the real behaviour is now stated too — a broken spawn yields a live-looking pid for a process that died immediately, and surfaces later as a confusing assertion. cmdline_of's comment needs BOTH edges of separator stripping ruled out, and the previous revision over-corrected: it deleted the trailing-space argument, which was true and load-bearing, leaving only the leading-edge one. Measured: with the trailing space a trailing tab survives, without it the same tab is stripped. Both edges are named again, each by its own property. docs/testing.md gained "like every other lane" while fixing the previous claim. The two native Windows lanes call --report-holders at neither point; their preflight failure path is Write-Host then exit with no PowerShell equivalent. Scoped to the bash lanes, with the exception stated. Also pins the decoy registration itself, which nothing else falsified, and describes it accurately: an untracked decoy is signalled by nothing and outlives even a GREEN run — the /workload fixture by its full 300s, since reclaim is meant to spare that one. The check's own limitation is stated rather than implied: an exact literal catches a tracking call removed, not a decoy added without one. Signed-off-by: Tomas Saaristola --- docs/testing.md | 4 ++- scripts/reclaim-gpu.sh | 61 +++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index fa8a41e21e..1586167c1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -855,7 +855,9 @@ with their root list pinned to the script's by an `xtask` contract test. The WSL lanes skip the proactive reclaim before preflight — they provision a fresh guest per job and unregister it afterwards, so no prior run's processes survive into them — but they still call the script for `--report-holders` when their -preflight fails, like every other lane. Its `--self-test` +preflight fails, as the other bash lanes do. The two native Windows lanes call +it at neither point: their preflight failure path has no PowerShell equivalent +of the diagnostic. Its `--self-test` needs no GPU — it spawns decoy processes and asserts which ones the matching rule selects — so it runs on the GitHub-hosted lane rather than only where GPU hardware is held, under the same `heavy` path filter as the other checks there diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 5433c81107..b00e49b9d0 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -148,14 +148,17 @@ process_alive() { # # `read` gives leftover words AND their intervening separators to the LAST name, # so a tab EMBEDDED in the command line lands in `cmdline` verbatim. Separators -# are stripped only where the assigned field BEGINS or ENDS with them, and the -# command line is argv[0] first — a path — so neither applies here. +# are stripped only where the assigned field BEGINS or ENDS with them, and BOTH +# edges are ruled out here — each by its own property, so both are named: # -# That is narrower than it first read. Leading separators of the trailing field -# are stripped too: `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`, not -# a tab then `x`. It would take an argv[0] that itself began with a tab to reach -# that, which no caller of this script produces — but "a tab can never be -# stripped" is not the true statement, and this comment said so for a revision. +# - leading: the command line starts with argv[0], a path, so the trailing +# field never begins with a separator. (Were it to, a leading tab WOULD be +# stripped: `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`.) +# - trailing: flattening the final NUL always leaves the string ending in a +# space, so a tab is never the last byte. This one is load-bearing and +# measured — with the trailing space `…arg\t ` survives intact, without it +# the same tab is stripped. The same property is relied on again at the +# self-test's record check; if it ever changes, both move together. # # An earlier revision also claimed the tab "shifts the field boundary" and # flattened it for that reason — also false, and deleting the tab from the @@ -652,11 +655,24 @@ self_test() { delimiter_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" # Each pid is registered with the EXIT trap the moment it exists, for the same - # reason the scratch-dir traps above are widened one statement at a time: every - # spawn below is fallible (mkdir, cp, chmod), and `set -e` aborts the function - # on the first failure. Registering them only in the list further down would - # leave every ALREADY-spawned decoy running with nothing arranged to kill it — - # including the stubborn one, which ignores SIGTERM and loops forever. + # reason the scratch-dir traps above are widened one statement at a time: + # registering them only in the list further down would leave every + # ALREADY-spawned decoy running with nothing arranged to kill it — including + # the stubborn one, which ignores SIGTERM and loops forever. Measured: a + # failing statement between two spawns left 3 survivors before this change and + # 0 after. + # + # NOT because `set -e` aborts on a failing spawn — it does not, and an earlier + # version of this comment said it did. `inherit_errexit` is off, so errexit + # does not reach inside `$( )`, and each helper ends in an `echo` that + # succeeds regardless; a hard failure injected into a spawn helper runs the + # suite to a green exit 0. The routes that DO reach the trap mid-way are a + # signal (a cancelled CI job), and any fallible statement between the spawns + # here — including one a later edit adds, which is the case this guards. + # + # The flip side is worth knowing: because a broken spawn does not abort, it + # yields a live-looking pid for a process that died immediately, and surfaces + # later as a confusing assertion failure rather than as "the spawn failed". prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" selftest_track_decoy "${prewarm_pid}" workload_pid="$(spawn_decoy "${workload_decoy}")" @@ -676,6 +692,27 @@ self_test() { # itself is absent deliberately: it is already dead, and killing its keeper is # what lets init reap it. Same set the trap holds, by construction. decoy_pids=("${SELFTEST_DECOY_PIDS[@]}") + # The registration itself, pinned. Without this, dropping a + # `selftest_track_decoy` call is silent AND leaks for real: both the EXIT trap + # and the final kill read this same list, so an untracked decoy is signalled + # by nothing. It survives even a GREEN run — measured, the `/workload` fixture + # outlives the suite by its full 300s, because `reclaim` is supposed to spare + # that one and the tracked list is the only other thing that would kill it. + # (An earlier version of this comment claimed such a decoy "is still killed at + # the end". It is not, and it is only incidentally true for the three fixtures + # `reclaim` itself signals.) + # + # LIMITATION, since an exact literal cannot carry its own maintenance: this + # catches a tracking call REMOVED from an existing decoy, where the count + # drops. It does NOT catch a decoy ADDED without one — the count stays at the + # stale literal and the run passes. Adding a spawn means bumping this number + # in the same edit; there is no mechanism enforcing that. + if [[ "${#SELFTEST_DECOY_PIDS[@]}" -ne 7 ]]; then + echo "FAIL: ${#SELFTEST_DECOY_PIDS[@]} decoys registered with the EXIT trap, expected 7" + echo " tracked: ${SELFTEST_DECOY_PIDS[*]}" + echo " a spawn is not being tracked; it would outlive the run, aborted or not" + failures=$((failures + 1)) + fi # Give the decoys a moment to appear in /proc with their full argv. sleep 1 From 51cba178e6c500acc1d76e76aa9a25fdfca32994 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 29 Sep 2026 09:15:09 +0000 Subject: [PATCH 14/16] ci(e2e): cover report_holders and the rm root, pin WSL diagnostics (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the five non-blocking items from the review at 20d65cc. report_holders and the bare invocation's rm -rf root were untested: the self-test now lists holders against the surviving bystanders (a marker is listed, a marker-less binary is not, a marker is matched literally) and asserts the TMPDIR fallback through a resolver split out of main, so it can be checked without deleting anything. Each new assertion was mutation-tested: dropping the marker derivation, switching -F to -E, over-broadening the filter, and removing the absolute-path guard each turn the self-test red. The two WSL --report-holders calls now cd into the guest clone like their sibling steps, instead of resolving a relative path against whatever cwd WSL inherits, which || true would hide. Check 8 names a dead stubborn decoy as such rather than as a guard that rejected a match. The trailing-space comment no longer says "always": measured, a process that overwrites argv AND its environment with no NUL gets an unterminated cmdline that can end in a tab — a miss, never a wrong kill. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 6 +- .github/workflows/nightly.yml | 6 +- scripts/reclaim-gpu.sh | 102 ++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 5b00403445..9e5acbb034 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -1227,7 +1227,11 @@ jobs: # Name what is holding the card instead of leaving the next reader to # reconstruct it: --report-holders prints per-process VRAM and the # engine processes visible here. Never fails the step itself. - bash scripts/reclaim-gpu.sh --report-holders || true + # From the guest clone, like every sibling step: `wsl -- bash ` + # gets no --cd, so a bare relative path would resolve against whatever + # directory WSL maps the Windows step's cwd to, and `|| true` would + # hide a miss. + (cd /root/work/rocm-cli && bash scripts/reclaim-gpu.sh --report-holders) || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1b15b35525..9e9092a396 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1353,7 +1353,11 @@ jobs: # Name what is holding the card instead of leaving the next reader to # reconstruct it: --report-holders prints per-process VRAM and the # engine processes visible here. Never fails the step itself. - bash scripts/reclaim-gpu.sh --report-holders || true + # From the guest clone, like every sibling step: `wsl -- bash ` + # gets no --cd, so a bare relative path would resolve against whatever + # directory WSL maps the Windows step's cwd to, and `|| true` would + # hide a miss. + (cd /root/work/rocm-cli && bash scripts/reclaim-gpu.sh --report-holders) || true exit 1 # This body runs under a bare `bash `: Invoke-WslBash.ps1 adds # `-eo pipefail` only when passed `-PipeFail`, which this step does diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index b00e49b9d0..efabfa1de9 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -154,11 +154,19 @@ process_alive() { # - leading: the command line starts with argv[0], a path, so the trailing # field never begins with a separator. (Were it to, a leading tab WOULD be # stripped: `IFS=$'\t' read -r pid cmdline <<<$'1\t\tx'` yields `x`.) -# - trailing: flattening the final NUL always leaves the string ending in a -# space, so a tab is never the last byte. This one is load-bearing and -# measured — with the trailing space `…arg\t ` survives intact, without it -# the same tab is stripped. The same property is relied on again at the -# self-test's record check; if it ever changes, both move together. +# - trailing: /proc//cmdline ends in a NUL for every process as execve +# left it — every engine this rule matches — and flattening that NUL +# leaves a trailing space, so a tab is never the last byte. This one is +# load-bearing and measured — with the trailing space `…arg\t ` survives +# intact, without it the same tab is stripped. The same property is relied +# on again at the self-test's record check; if it ever changes, both move +# together. +# +# Not every process, though. One that rewrites its argv AND environment +# with no NUL left in either (setproctitle taken to the limit) gets a +# cmdline with no terminator at all — measured, it can end in a tab. The +# tab is then stripped, the identity check below fails, and the pid is +# reported "recycled" and not signalled: a miss, never a wrong kill. # # An earlier revision also claimed the tab "shifts the field boundary" and # flattened it for that reason — also false, and deleting the tab from the @@ -596,6 +604,7 @@ self_test() { local forced_out guard_pid local escapee_pid escapee_cmd guard_rc probe_cmd local zombie_pid zombie_keeper_pid + local holders_out holders_engines tmp_case local decoy_pids local superseded_hit=0 local probe_failures=0 @@ -956,8 +965,16 @@ self_test() { # The COMPARISON is covered here; check 9 covers the two CALL SITES by # forcing the verdict, because a pid cannot be made to be reused by a # different process on demand. - if same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")"; then + # The accept case needs the stubborn decoy alive to be comparable at all. If + # it has already exited, both reads fail and the guard answers "gone" — a + # dead fixture, not a guard that rejects a match, so it is reported as one. + guard_rc=0 + same_selected_process "${stubborn_pid}" "$(cmdline_of "${stubborn_pid}")" || guard_rc=$? + if [[ "${guard_rc}" == 0 ]]; then echo "ok: escalation guard accepts an unchanged command line" + elif [[ "${guard_rc}" == 2 ]]; then + echo "FAIL: SIGTERM-ignoring decoy (pid=${stubborn_pid}) exited before the escalation guard could be checked" + failures=$((failures + 1)) else echo "FAIL: escalation guard rejected an unchanged command line; nothing would escalate" failures=$((failures + 1)) @@ -1106,6 +1123,51 @@ self_test() { echo "ok: ordinary decoy exited on SIGTERM without escalation" fi + # 12. report_holders' engine filter — the diagnostic every bash lane runs on a + # failed preflight. The bystanders survived check 10, so they are still + # there to be listed: the manual serve carries an engine marker and must + # appear, the harness binary carries none and must not. Deleting the + # marker derivation leaves grep with no pattern at all, which lists + # nothing and fails the first assertion. + holders_out="$(report_holders 2>&1)" + holders_engines="$(sed -n '/^--- engine\/serve processes/,/^--- of those/p' <<<"${holders_out}")" + if awk -v p="${workload_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "ok: report_holders lists a process carrying an engine marker" + else + echo "FAIL: report_holders did not list the manual serve (pid=${workload_pid})" + failures=$((failures + 1)) + fi + if awk -v p="${harness_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "FAIL: report_holders listed a process with no engine marker (pid=${harness_pid})" + failures=$((failures + 1)) + else + echo "ok: report_holders leaves out a process with no engine marker" + fi + # Fixed-string, not regex: under -E the `.` in this marker would match the + # `-` in the manual serve's path and list it. None of the real markers + # holds a metacharacter, so only an override can tell the two apart. + holders_out="$(ENGINE_MARKERS=('manual-serve/llama.server') && report_holders 2>&1)" + holders_engines="$(sed -n '/^--- engine\/serve processes/,/^--- of those/p' <<<"${holders_out}")" + if awk -v p="${workload_pid}" '$1 == p { found = 1 } END { exit !found }' <<<"${holders_engines}"; then + echo "FAIL: report_holders matched a marker as a regex, not a literal substring" + failures=$((failures + 1)) + else + echo "ok: report_holders matches markers as literal substrings" + fi + + # 13. The bare invocation's rm -rf root. Asserted on the resolver rather than + # by running the deletion, which would clear the host's real scenario + # dirs. A relative TMPDIR must fall back to /tmp — the one branch whose + # failure is `rm -rf` against the CWD. + for tmp_case in 'relative/dir|/tmp' '|/tmp' '/abs/dir/|/abs/dir' '/abs/dir|/abs/dir'; do + if [[ "$(TMPDIR="${tmp_case%%|*}" scenario_tmp_root)" == "${tmp_case#*|}" ]]; then + echo "ok: TMPDIR='${tmp_case%%|*}' clears under ${tmp_case#*|}" + else + echo "FAIL: TMPDIR='${tmp_case%%|*}' did not resolve to ${tmp_case#*|}" + failures=$((failures + 1)) + fi + done + # Includes prewarm_pid: on a GREEN run reclaim has already killed it, but on a # FAILED run it was not selected, and it would otherwise outlive the scratch # tree for its full 300s as an orphan. Killing the zombie's keeper lets init @@ -1119,8 +1181,23 @@ self_test() { echo "reclaim-gpu self-test: all checks passed" } +# The directory the bare invocation clears scenario temp dirs under: TMPDIR when +# it is absolute, /tmp otherwise. Printed without a trailing slash. +# +# The absolute-path test is the SAME one the roots above apply, and it matters +# most here: this feeds the only consumer that deletes rather than merely +# failing to match. Interpolated raw, a relative TMPDIR makes the caller's +# `rm -rf relative/dir/rocm-e2e-*` resolve against the CWD — the repo checkout, +# on a CI runner. No lane sets one, which is a reason to skip the value, not a +# reason to hand it to `rm -rf` unchecked. A function rather than inline so the +# self-test can assert the fallback without deleting anything. +scenario_tmp_root() { + local root="${TMPDIR:-}" + [[ "${root}" == /* ]] || root='/tmp' + printf '%s' "${root%/}" +} + main() { - local scenario_tmp case "${1:-}" in '') reclaim 0 @@ -1133,16 +1210,7 @@ main() { # the guard was dead on the one lane that redirects. Both are listed # rather than just TMPDIR: the lanes that do not set it still want /tmp, # and a stale tree from before a redirect was added would outlive it. - # - # The absolute-path test is the SAME one the roots above apply, and it - # matters most here: this is the only consumer that deletes rather than - # merely failing to match. Interpolated raw, a relative TMPDIR makes this - # `rm -rf relative/dir/rocm-e2e-*` resolved against the CWD — the repo - # checkout, on a CI runner. No lane sets one, which is a reason to skip - # the value, not a reason to hand it to `rm -rf` unchecked. - scenario_tmp="${TMPDIR:-}" - [[ "${scenario_tmp}" == /* ]] || scenario_tmp='/tmp' - rm -rf "${scenario_tmp%/}"/rocm-e2e-* /tmp/rocm-e2e-* 2>/dev/null || true + rm -rf "$(scenario_tmp_root)"/rocm-e2e-* /tmp/rocm-e2e-* 2>/dev/null || true ;; --dry-run) reclaim 1 ;; --report-holders) report_holders ;; From 96f65917209a7be825638b838e2e12d0c32bf9dd Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 29 Sep 2026 12:54:22 +0000 Subject: [PATCH 15/16] ci(e2e): stop promising a /workload serve is never touched (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blocking findings from the review at 51cba17 are claims the diff's own code contradicts. Each was confirmed by measurement first. Seven workflow comments said reclaim is "scoped to E2E-owned trees, so a manual /workload serve on a shared self-hosted runner is never touched". The roots are unanchored substrings of the whole command line, so a manual serve IS selected when an argument happens to name one: `/workload/manual/llama-server --model-path /home/dev/e2e-shared-models/x.gguf` matches and is TERMed then KILLed, measured against the rule directly. The script's header already documented that exposure, so the comments were overriding the honest text they pointed at. They now state what the AND rule actually provides. The header's own unqualified sentence is fixed too — it is where the seven now send the reader, and the qualification sat thirty lines below it. A stale "promise below" pointing upward goes with it. reclaim_script_roots() located the array's closing paren in raw text before stripping comments, and accepted single quotes only, one root per line. Three legal spellings of the identical array broke it, and the resulting message blamed the PowerShell mirrors: an inline comment containing a paren made the test demand the deletion of `e2e-prewarm` from both mirrors — the root this change exists to add, and following that advice would silently drop Windows coverage while turning the suite green. Comments now come off before the paren is located, both quotings are accepted, and every non-empty line inside the array must parse into exactly one root, so an unrecognised spelling fails as a parser that did not understand the line rather than as fabricated drift. A parsed root may not contain its own delimiter, which is what rejects the whole array folded onto one line. Not tokenised on whitespace: that would split a future root containing a space into two bogus roots, the same defect class in a new place. All five mutations re-run: the three spellings now parse or fail honestly, and real drift in both directions still fails with the message it had. Four non-blocking items close with it. The summary line says "signalled" rather than "terminated", because the count is incremented where TERM is sent, both kill calls swallow their errors, and the SIGKILL loop never touches it. Self-test checks 4 and 5 report SKIP instead of a vacuous ok when the selection is empty. A new check asserts the bare invocation's rm -rf still resolves its root through scenario_tmp_root, closing a covered helper nothing was obliged to call. The contract test names the hardcoded /tmp/ prefix as the one absolute prefix its mapping knows. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 20 ++++--- .github/workflows/nightly.yml | 15 +++-- scripts/reclaim-gpu.sh | 58 ++++++++++++++++--- xtask/src/workflow_contract.rs | 83 ++++++++++++++++++++++++---- 4 files changed, 143 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 9e5acbb034..e2b363e3c4 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -180,8 +180,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh @@ -404,8 +405,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh @@ -1402,8 +1404,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh @@ -1529,8 +1532,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9e9092a396..ee339f4f05 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -373,8 +373,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh @@ -496,8 +497,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh @@ -589,8 +591,9 @@ jobs: # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine # process on the card and starves this job's serves. Scoped to E2E-owned - # trees, so a manual /workload serve on a shared self-hosted runner is - # never touched. See scripts/reclaim-gpu.sh. + # trees: a manual /workload serve is matched only if its command line + # also names an E2E root — see the exposure documented in + # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes run: bash scripts/reclaim-gpu.sh diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index efabfa1de9..e5172667ac 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -23,9 +23,16 @@ # # The rule here is order-independent and covers every E2E-owned tree: a process # is reclaimed when its command line names an E2E root AND an engine/serve -# process. Both halves are required, which is what keeps a legitimate -# `/workload` manual-testing serve on a shared self-hosted runner safe — it -# names no E2E root. +# process. Both halves are required, which is what spares a legitimate +# `/workload` manual-testing serve on a shared self-hosted runner in the +# ordinary case: it names no E2E root, so the root half does not hold. +# +# That is a narrowing, NOT a guarantee, and it is stated here rather than only +# in WHAT THE ANCHORING DOES AND DOES NOT BUY below, because a reader who stops +# at this paragraph is the operator deciding whether their own serve is safe. A +# `/workload` serve IS selected when its command line happens to name a root in +# an argument — `--model-path /home/dev/e2e-shared-models/x.gguf` is the +# measured case. Read that section before relying on this one. set -euo pipefail @@ -53,8 +60,8 @@ set -euo pipefail # SIGKILLed. Measured, and briefly shipped, which is why it is spelled out. # # Anchoring the scenario root to an absolute path NARROWS that surface. It does -# not close it, and the `/workload` promise below is correspondingly weaker than -# it reads: +# not close it, and the `/workload` narrowing stated at the top of this file is +# correspondingly weaker than it would read on its own: # # - under a redirected TMPDIR, an argument naming a sibling path still # matches — `--extra-data-dir /tmp/mydir/rocm-e2e-notes` with @@ -136,7 +143,7 @@ process_alive() { # emits a record that splits into two on the way back: the real pid arrives # carrying a truncated command line, fails the identity check against its own # full one, and is reported "recycled ... not signalling" — so the leak survives -# both TERM and KILL while the summary prints "0 process(es) terminated". That is +# both TERM and KILL while the summary prints "0 process(es) signalled". That is # the silent miss this whole script exists to end, so it is fixed at the single # point every consumer already reads through rather than at each of them. # @@ -344,7 +351,14 @@ reclaim() { esac done <<<"${selected}" - echo "reclaim: ${killed} process(es) terminated" + # SIGNALLED, not terminated. The count is incremented where TERM is sent, and + # both kill calls are `2>/dev/null || true` so a signal this process was not + # permitted to send is invisible here; the SIGKILL loop does not touch the + # count at all. Claiming termination would therefore report success for a + # process still holding the card — the exact failure mode the unconditional + # `reclaimed` line had, which is what let this go unnoticed for hours. The + # per-process lines above are where the detail lives. + echo "reclaim: ${killed} process(es) signalled" } # Diagnostics for a preflight that hit its ceiling. Never fails: it runs on the @@ -776,7 +790,7 @@ self_test() { # then arrives with a TRUNCATED command line, fails the identity check # against its own full one, and is passed over as "recycled": the leak # survives TERM and KILL alike while the run signs off with "0 process(es) - # terminated". cmdline_of flattens it for that reason, and deleting that + # signalled". cmdline_of flattens it for that reason, and deleting that # flattening fails this check. # - a TAB does NOT, even though it is the field separator, because `read` # hands leftover separators to the last name and the trailing space left @@ -855,10 +869,24 @@ self_test() { echo "ok: every selection record survives delimiters in a command line" fi + # Checks 4 and 5 are NEGATIVE assertions, and a negative assertion is + # satisfied trivially by a selection that is empty. Both therefore gate their + # affirmation on the selection being non-empty, the same way check 7 gates its + # containment line and for the same reason: checks 2 and 6 already fail such a + # run, so this is not a false green — but "ok: a manual serve is not selected" + # printed on a run where nothing was selected at all is evidence that misleads + # whoever is diagnosing it. + # + # Reported as SKIP rather than counted as a failure, so the run's cause stays + # the check that actually detected the breakage rather than these two + # restating it. + # 4. A manual-testing serve is left alone (engine marker, but no E2E root). if grep -q "^${workload_pid} " <<<"${selected}"; then echo "FAIL: /workload manual serve was selected; reclaim must not touch it" failures=$((failures + 1)) + elif [[ -z "${selected}" ]]; then + echo "SKIP: /workload manual serve was not selected, but the selection is EMPTY — this check proved nothing" else echo "ok: /workload manual serve is not selected" fi @@ -868,6 +896,8 @@ self_test() { if grep -q "^${harness_pid} " <<<"${selected}"; then echo "FAIL: E2E test binary was selected; the engine half of the rule is not enforced" failures=$((failures + 1)) + elif [[ -z "${selected}" ]]; then + echo "SKIP: the E2E test binary was not selected, but the selection is EMPTY — this check proved nothing" else echo "ok: an E2E root without an engine marker is not selected" fi @@ -1167,6 +1197,18 @@ self_test() { failures=$((failures + 1)) fi done + # The resolver is only half of it. Nothing above asserts that the `rm -rf` + # statement still CALLS it, so reverting that one line to an inline + # `${TMPDIR:-/tmp}` reintroduces the relative-TMPDIR hazard with the whole + # suite green — a covered helper nothing is obliged to use. Asserted + # against this file's own source, because actually running the deletion is + # what the paragraph above rules out. + if grep -qE '^[[:space:]]*rm -rf "\$\(scenario_tmp_root\)"' "${BASH_SOURCE[0]}"; then + echo "ok: the bare invocation clears its scenario dirs through scenario_tmp_root" + else + echo "FAIL: the rm -rf statement no longer calls scenario_tmp_root; TMPDIR handling is bypassed" + failures=$((failures + 1)) + fi # Includes prewarm_pid: on a GREEN run reclaim has already killed it, but on a # FAILED run it was not selected, and it would otherwise outlive the scratch diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 483f974f57..90272fce51 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2082,22 +2082,74 @@ permissions: let text = std::fs::read_to_string(&p) .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) .replace("\r\n", "\n"); - let body = text + let after = text .split_once("E2E_ROOTS=(") .unwrap_or_else(|| panic!("{} must declare E2E_ROOTS=(", p.display())) - .1 + .1; + + // Comments come off BEFORE the closing `)` is located, not per line + // afterwards. An inline comment containing a `)` — `'/tmp/rocm-e2e' + // # see docs (section 3)` — otherwise closes the array at that paren, + // and every root below it vanishes from the parse. The test then fails + // in the mirror->script direction naming a root that is plainly still + // in the script, and a maintainer who follows that message deletes a + // real root from both PowerShell mirrors. Measured: that spelling made + // this test demand the deletion of `e2e-prewarm`, the root this whole + // change exists to add. + let decommented = after + .lines() + .map(|l| { + if l.trim_start().starts_with('#') { + "" + } else { + strip_comment(l) + } + }) + .collect::>() + .join("\n"); + let body = decommented .split_once(')') .unwrap_or_else(|| panic!("{} has an unterminated E2E_ROOTS array", p.display())) .0; - let roots: Vec = body - .lines() - .filter_map(|l| { - let l = strip_comment(l).trim(); - l.strip_prefix('\'') - .and_then(|l| l.strip_suffix('\'')) - .map(str::to_owned) - }) - .collect(); + + // Every non-empty line inside the array must parse into exactly one + // root. This is the load-bearing half: without it an unrecognised + // spelling is silently DROPPED, and a dropped root is indistinguishable + // from mirror drift downstream — so the failure arrives as a confident + // wrong remedy rather than as "this parser did not understand the + // array". A checker that converts a formatting change into advice that + // removes Windows coverage is worse than no checker on that axis. + // + // Deliberately NOT tokenised on whitespace: that would silently split a + // future root containing a space into two bogus roots, which is this + // same defect class in a new place. + let mut roots: Vec = Vec::new(); + for line in body.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + // Both bash quotings are accepted. A quoted root can never contain + // its OWN delimiter, so requiring that is what rejects the whole + // array folded onto one line — `E2E_ROOTS=('a' 'b')` parses as the + // single bogus root `a' 'b` under a prefix/suffix strip alone, and + // then fails script->mirror naming a root nobody wrote. + let parsed = ['\'', '"'].into_iter().find_map(|q| { + line.strip_prefix(q) + .and_then(|l| l.strip_suffix(q)) + .filter(|inner| !inner.contains(q)) + }); + let Some(root) = parsed else { + panic!( + "{}: E2E_ROOTS line `{line}` is not a single quoted root. This test's \ + parser did not understand it — that is NOT drift against the PowerShell \ + mirrors, so do not remove anything from them. Restore one quoted root \ + per line, or teach this parser the new spelling (EAI-8751)", + p.display() + ); + }; + roots.push(root.to_owned()); + } assert!( !roots.is_empty(), "{} declared no E2E_ROOTS entries — the parser or the array shape changed", @@ -2183,6 +2235,15 @@ permissions: // What it still pins is the part that matters here: that every root // the script knows about is named in both mirrors, so a root added // to one side cannot silently miss the Windows lanes. + // + // CAVEAT: the `/tmp/` prefix below is the only absolute prefix this + // mapping knows. A future root anchored under any other one — say + // `/var/tmp/rocm-e2e` — would be compared to the mirrors verbatim, + // fail against their bare segment, and need this line edited by + // hand: the manual sync this test exists to remove, reappearing one + // level up. Named rather than generalised, because there is exactly + // one anchored root today and a speculative prefix list would be + // untested code. let expected: Vec<&str> = roots .iter() .map(|r| r.strip_prefix("/tmp/").unwrap_or(r)) From 2a665ae1ded0c000fd0dbd5f6bfc5c90cee4377e Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 29 Sep 2026 13:39:05 +0000 Subject: [PATCH 16/16] ci(e2e): carry the signalled/terminated fix to the Windows mirrors (EAI-8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh re-read of the previous commit found it had fixed one instance of a claim rather than the class. Renaming the bash summary line to "signalled" left both PowerShell reclaims printing "N process(es) terminated" — and the rationale for the rename applies to them verbatim: `Stop-Process -ErrorAction SilentlyContinue` swallows a failure exactly as `kill ... 2>/dev/null || true` does. The Windows count is weaker still, being `$leaked.Count` — the SELECTED set, with no identity re-verification and no record of which stops succeeded — so a process that survived is counted as terminated. Grepped the whole repo for the claim this time, not just the two files the diff already touched; no other surface restates it. The E2E_ROOTS parser is split into a pure `parse_e2e_roots(text, origin)`, which is what makes its edge cases testable at all. Every spelling behind the previous commit was found by editing the tracked script by hand, and none of them could fail CI afterwards — the same "covered but not exercised" gap that check 13's call-site assertion closed on the bash side. Three tests now pin it: the legal respellings that must parse identically (inline comment containing a paren, either quoting, full-line and blank lines inside the array, a comment above it quoting the declaration's own text, a root containing a space), the spellings that must be REJECTED as the parser's own misunderstanding rather than as mirror drift, and the empty array. Each asserts the message does not send a reader to delete a root from Windows. Anchoring now happens after comment-stripping, not before, so a header comment quoting the literal `E2E_ROOTS=(` cannot redirect the parse into prose. That case failed loudly rather than silently even before — the per-line completeness assertion already contained it — but it failed on a file that was correct. `strip_comment` is not quote-aware, so a root whose value contains " #" arrives truncated. Left as a loud rejection and named in the message rather than fixed: making a helper every other caller depends on quote-aware, for a spelling no root uses, would be untested code on a shared path. Mutations re-run: dropping the own-delimiter guard, silently skipping an unparsable line, and anchoring before decommenting each red the test that covers them. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 7 +- .github/workflows/nightly.yml | 7 +- xtask/src/workflow_contract.rs | 192 +++++++++++++++++++++++---- 3 files changed, 181 insertions(+), 25 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index e2b363e3c4..f30cfcc408 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -691,7 +691,12 @@ jobs: Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaim: $(@($leaked).Count) process(es) terminated" + # SIGNALLED, not terminated, for the same reason the bash summary says + # so: -ErrorAction SilentlyContinue swallows a Stop-Process that could + # not kill its target, and this count is the SELECTED set, so a + # process that survived is still counted. Weaker than the bash line, + # which at least counts only what it re-verified before signalling. + Write-Host "reclaim: $(@($leaked).Count) process(es) signalled" # GPU preflight (best-effort on Windows): bounded wait for the GPU to be # available so a wedged/held GPU fails fast instead of hanging to the cap. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ee339f4f05..dd89dda9e9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -914,7 +914,12 @@ jobs: Write-Host "reclaim: terminating pid=$($p.ProcessId) cmd=$($p.CommandLine)" Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue } - Write-Host "reclaim: $(@($leaked).Count) process(es) terminated" + # SIGNALLED, not terminated, for the same reason the bash summary says + # so: -ErrorAction SilentlyContinue swallows a Stop-Process that could + # not kill its target, and this count is the SELECTED set, so a + # process that survived is still counted. Weaker than the bash line, + # which at least counts only what it re-verified before signalling. + Write-Host "reclaim: $(@($leaked).Count) process(es) signalled" # Best-effort when rocm-smi is absent; when present, fail within the bounded # ceiling instead of letting a held or wedged GPU consume the full job cap. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 90272fce51..b757d4ae55 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2082,21 +2082,36 @@ permissions: let text = std::fs::read_to_string(&p) .unwrap_or_else(|e| panic!("reading {}: {e}", p.display())) .replace("\r\n", "\n"); - let after = text - .split_once("E2E_ROOTS=(") - .unwrap_or_else(|| panic!("{} must declare E2E_ROOTS=(", p.display())) - .1; - - // Comments come off BEFORE the closing `)` is located, not per line - // afterwards. An inline comment containing a `)` — `'/tmp/rocm-e2e' - // # see docs (section 3)` — otherwise closes the array at that paren, - // and every root below it vanishes from the parse. The test then fails - // in the mirror->script direction naming a root that is plainly still - // in the script, and a maintainer who follows that message deletes a - // real root from both PowerShell mirrors. Measured: that spelling made - // this test demand the deletion of `e2e-prewarm`, the root this whole - // change exists to add. - let decommented = after + parse_e2e_roots(&text, &p.display().to_string()) + } + + /// The parsing half of [`reclaim_script_roots`], split out so every spelling + /// it must cope with is a standing test rather than a mutation somebody has + /// to remember to run by hand against the tracked script. + /// + /// That split is the point. The previous version of this parser could only + /// be exercised by editing `scripts/reclaim-gpu.sh` itself, so three legal + /// spellings of the same array reached review unnoticed — and the failure + /// they produced blamed the PowerShell mirrors, advising a maintainer to + /// delete a real root from Windows. + fn parse_e2e_roots(text: &str, origin: &str) -> Vec { + // Comments come off BEFORE anything is located — before the array's + // START as well as its closing `)`. Two distinct mis-parses, one fix: + // + // - an inline comment containing a `)` — `'/tmp/rocm-e2e' # see docs + // (section 3)` — closes the array at that paren, so every root below + // it vanishes from the parse. The test then fails in the + // mirror->script direction naming a root that is plainly still in + // the script, and a maintainer who follows that message deletes a + // real root from both PowerShell mirrors. Measured: that spelling + // made this test demand the deletion of `e2e-prewarm`, the root this + // whole change exists to add. + // - a comment ABOVE the array that merely quotes the literal text + // `E2E_ROOTS=(` would otherwise anchor the search inside that + // sentence rather than at the declaration. This file already quotes + // bash array syntax in prose (`E2E_ROOTS+=(` appears in a comment), + // so that is a spelling a future header edit can reach. + let decommented = text .lines() .map(|l| { if l.trim_start().starts_with('#') { @@ -2108,8 +2123,11 @@ permissions: .collect::>() .join("\n"); let body = decommented + .split_once("E2E_ROOTS=(") + .unwrap_or_else(|| panic!("{origin} must declare E2E_ROOTS=(")) + .1 .split_once(')') - .unwrap_or_else(|| panic!("{} has an unterminated E2E_ROOTS array", p.display())) + .unwrap_or_else(|| panic!("{origin} has an unterminated E2E_ROOTS array")) .0; // Every non-empty line inside the array must parse into exactly one @@ -2140,24 +2158,152 @@ permissions: .filter(|inner| !inner.contains(q)) }); let Some(root) = parsed else { + // `strip_comment` splits on a literal " #" with no quote + // awareness, so a root whose VALUE contains that sequence — + // `'/tmp/e2e root #2'`, which bash itself accepts, since `#` is + // inert inside single quotes — arrives here already truncated + // and unterminated. Named in the message rather than fixed: + // this lands as a loud false failure, never a silent miss, and + // making the shared helper quote-aware for a spelling no root + // uses would be untested code on a path every other caller + // depends on. panic!( - "{}: E2E_ROOTS line `{line}` is not a single quoted root. This test's \ - parser did not understand it — that is NOT drift against the PowerShell \ - mirrors, so do not remove anything from them. Restore one quoted root \ - per line, or teach this parser the new spelling (EAI-8751)", - p.display() + "{origin}: E2E_ROOTS line `{line}` is not a single quoted root. This \ + test's parser did not understand it — that is NOT drift against the \ + PowerShell mirrors, so do not remove anything from them. Restore one \ + quoted root per line, or teach this parser the new spelling. Note the \ + line is shown after comment-stripping, so a root whose value contains \ + \" #\" appears truncated here (EAI-8751)" ); }; roots.push(root.to_owned()); } assert!( !roots.is_empty(), - "{} declared no E2E_ROOTS entries — the parser or the array shape changed", - p.display() + "{origin} declared no E2E_ROOTS entries — the parser or the array shape changed" ); roots } + /// An `E2E_ROOTS` array spelled the way the script spells it today, with + /// enough surrounding file to exercise the comment handling. + #[cfg(test)] + fn roots_fixture(body: &str) -> String { + format!( + "#!/usr/bin/env bash\n# a header comment\nset -euo pipefail\nE2E_ROOTS=(\n{body})\nENGINE_MARKERS=(\n 'llama-server'\n)\n" + ) + } + + /// Every spelling this parser must cope with, as a test rather than as a + /// mutation of the tracked script. Each of these was found by editing + /// `scripts/reclaim-gpu.sh` by hand during review; none of them could fail + /// CI afterwards, which is the gap this closes. + #[test] + fn e2e_roots_parser_accepts_legal_respellings_of_the_same_array() { + let canonical = vec!["/tmp/rocm-e2e".to_owned(), "e2e-shared".to_owned()]; + + // As written today. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e'\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // An inline comment containing a `)`. This closed the array early and + // made the test demand the deletion of a real root from Windows. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e' # see docs (section 3)\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // Double quotes, and a mix of both. Silently dropped the entry before. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" \"/tmp/rocm-e2e\"\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // A full-line comment inside the array, indented or at column 0, and a + // blank line. All three are skipped rather than failing completeness. + assert_eq!( + parse_e2e_roots( + &roots_fixture(" '/tmp/rocm-e2e'\n# flush left\n # indented\n\n 'e2e-shared'\n"), + "fixture" + ), + canonical + ); + + // A comment ABOVE the array quoting the declaration's own text. The + // anchor must find the declaration, not the sentence about it. + let text = "#!/usr/bin/env bash\n# see how E2E_ROOTS=( is declared below\nE2E_ROOTS=(\n '/tmp/rocm-e2e'\n 'e2e-shared'\n)\n"; + assert_eq!(parse_e2e_roots(text, "fixture"), canonical); + + // A root containing a space is preserved whole, which is what rules out + // tokenising the body on whitespace. + assert_eq!( + parse_e2e_roots(&roots_fixture(" '/tmp/rocm e2e'\n"), "fixture"), + vec!["/tmp/rocm e2e".to_owned()] + ); + } + + /// The other half: spellings the parser must REJECT, and reject as its own + /// misunderstanding rather than as drift against the PowerShell mirrors — + /// because the mirror-drift message tells a maintainer to delete a root + /// from Windows, and following it on a false positive is what makes this + /// worse than having no checker. + #[test] + fn e2e_roots_parser_rejects_unparsable_lines_without_blaming_the_mirrors() { + for (label, body) in [ + ( + "array folded onto one line", + " '/tmp/rocm-e2e' 'e2e-shared'\n", + ), + ("unquoted entry", " e2e-shared\n"), + ("unterminated quote", " '/tmp/rocm-e2e\n"), + // `strip_comment` is not quote-aware, so this arrives truncated. + // Rejected loudly, never silently dropped. + ("value containing \" #\"", " '/tmp/rocm e2e #2'\n"), + ] { + let text = roots_fixture(body); + let err = std::panic::catch_unwind(|| parse_e2e_roots(&text, "fixture")) + .expect_err(&format!("{label} must not parse")); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("did not understand"), + "{label} failed with the wrong message — it must not read as mirror drift: {msg}" + ); + assert!( + msg.contains("do not remove anything from them"), + "{label} must tell the reader NOT to edit the mirrors: {msg}" + ); + } + } + + /// An empty array is a parser/shape change, not zero roots — and must not + /// sail through as "no roots declared, nothing to compare". + #[test] + fn e2e_roots_parser_rejects_an_empty_array() { + let err = std::panic::catch_unwind(|| parse_e2e_roots(&roots_fixture(""), "fixture")) + .expect_err("an empty E2E_ROOTS must not parse as success"); + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!(msg.contains("declared no E2E_ROOTS entries"), "got: {msg}"); + } + /// The root half of each PowerShell reclaim: the FIRST `-match '…'` /// alternation on the `Where-Object` line. ///