diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8c957cc720..0e065d02e9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,3 +8,7 @@ SPDX-License-Identifier: MIT - [ ] If this PR adds a new subcommand or subsystem, its domain implementation lives in its own file per `docs/architecture.md` (the clap declaration in `cli.rs` and the dispatch wiring in `main.rs`/`lib.rs` are expected, not a violation). - [ ] If this PR adds, renames or removes a production source file (or a `.rs` file under `xtask/src` or `tests/e2e-cucumber`), `docs/architecture.md` has the matching entry (`cargo xtask check-architecture-doc` fails otherwise), and the prose of any entry whose file changed role still describes it, which the check cannot verify. - [ ] Every new or changed user-facing message was read against the code path that runs after it, and its test asserts the resulting state — not only the wording — per AGENTS.md §3. Covers claims of an outcome, remediation advice naming a command, and promises that something will *not* happen. +- [ ] Comments and doc lines this PR adds or changes that state how the code behaves were checked against that code, and each that is the reason for a decision names the file and the function, constant or test that makes it true (AGENTS.md §3). +- [ ] Any existing scenario this PR moves off the blocking mock `e2e` job (for example by a tag) is justified as a deletion (AGENTS.md §8). +- [ ] Commits answering review feedback were diffed whole against the reviewed head and widen no documented user-facing option's meaning or authority (AGENTS.md §11). +- [ ] Every commit message on the branch describes what its own commit does in words still true at the branch head (AGENTS.md §5). diff --git a/.github/scripts/Invoke-WslBash.ps1 b/.github/scripts/Invoke-WslBash.ps1 index 8770126675..fa07f64cc3 100644 --- a/.github/scripts/Invoke-WslBash.ps1 +++ b/.github/scripts/Invoke-WslBash.ps1 @@ -1,3 +1,7 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + # Runs a bash script inside the pinned WSL guest. # # Piping script text to `wsl.exe` from Windows PowerShell 5.1 is unreliable: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230544bfa0..489428cbb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1020,10 +1020,15 @@ jobs: # serve-33/serve-34/serve-35 (the native-Windows vLLM refusal for `serve` and # `engines install`) are folded in the same way: `@requires-os:windows`, # hermetic, and otherwise run only on the optional self-hosted lane. + # + # runtime-lifecycle-19/20/21 (runtimes installed before and after the ROCm + # 10 layout, side by side) are folded in for the same reason: planted + # runtimes, no GPU or network, and their Windows paths are what this + # lane proves. env: E2E_INCLUDE_LIFECYCLE: "1" E2E_ONLY_LIFECYCLE: "1" - E2E_ALSO_RUN_IDS: "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows" + E2E_ALSO_RUN_IDS: "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows,runtime-lifecycle-side-by-side-across-packaging-change,runtime-lifecycle-side-by-side-uninstall-keeps-the-others,runtime-lifecycle-side-by-side-shared-name-refused" run: | # Derive from CARGO_TARGET_DIR when it is set, falling back to the # default target dir, so a future job-level CARGO_TARGET_DIR cannot @@ -1869,8 +1874,10 @@ jobs: # e2e-gpu-strix-windows). They live there — not here — so a job queued on an # offline self-hosted runner (which GitHub cannot cancel) can never hold this # workflow's concurrency group and stall its merge-required checks (EAI-7548). - # They are non-blocking (continue-on-error) and fork-safe (self-hosted runners - # do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. + # They are non-blocking (none is a required check) and fork-restricted: GitHub does + # not start self-hosted jobs from an external fork's pull_request run without + # approval (whether that is every fork run or only a first-time contributor's + # is a repository setting). See docs/ci-hardware-testing.md. # ── E2E tests (cucumber-rs) ─────────────────────────────────────────────── # diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 96b63f1b9c..9ed5acc279 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -9,9 +9,11 @@ name: E2E self-hosted # THIS workflow's supersession, never the required checks in ci.yml. Only # pull_request runs are superseded (see `concurrency:` below). # -# Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, -# MI300X: the only lane where vLLM is the effective engine) and one Windows GPU -# pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix +# Per-PR scope: a pull request runs one Linux GPU silicon, MI300X, as two checks +# (`e2e-gpu-queue`, the primary, on the shared GPU queue, and `e2e-gpu`, the +# second, on a self-hosted runner; vLLM is the effective engine on both, and no +# other silicon here runs it) and one Windows GPU pool (the Strix Halo pool: +# `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix # Ubuntu lane keeps its push / dispatch triggers; it does not start on # pull_request or merge_group. There is no W7900 lane here: W7900 is covered only # by the scheduled queue nightly's `w7900-dual` lane (docs/nightly-on-the-queue.md), @@ -43,7 +45,9 @@ on: # every GPU lane skips because `changes` skips. This frees runners only where # the cancel completes: a job queued for a runner that is not picking it up # (a lane whose runner is busy) may not finish cancelling, so this - # does NOT clear such queued runs. + # does NOT clear such queued runs. Where the MI300X job is the one that + # reaches its runner, its stale-run guard ends it and `cancel-stale-run` + # cancels the run (same-repo PRs only); the Strix lanes carry no guard. types: [opened, synchronize, reopened, closed] merge_group: branches: [main] @@ -205,7 +209,9 @@ jobs: # Exists only to start a run in the PR's concurrency group when the PR is # merged or closed: that run cancels the PR's older run. A GPU job of that run # which is queued for a runner that is not picking it up may not finish - # cancelling (observed on queued runs). + # cancelling (observed on queued runs). A same-repo PR's run whose MI300X job + # reaches its runner is then cancelled by `e2e-gpu`'s stale-run guard and + # `cancel-stale-run`. pr-closed: name: Supersede runs of a closed PR if: github.event_name == 'pull_request' && github.event.action == 'closed' @@ -221,9 +227,18 @@ jobs: # `mi300x`, not `amd-gpu`: every AMD GPU runner carries `amd-gpu`, including # the Strix Halo hosts, so `[self-hosted, linux, amd-gpu]` let this lane land # on gfx1151 — red on the WSL host, and (worse) a silent pass published as - # `e2e-gpu-report`, which the consolidated grid renders as MI300X. + # an MI300X report, which the consolidated grid renders as MI300X. + # + # On a pull request this lane is the second MI300X check: the first is the + # queue run of `e2e-gpu-queue` below, which does not wait for one of these + # runners. This lane is kept for the runners' own coverage (their runtime and + # cache), and its failure is as non-blocking as it always was. runs-on: [self-hosted, linux, mi300x] needs: [changes] + # Read by `e2e-report`: a run that skipped itself as stale uploads nothing, + # which is not a missing report. + outputs: + stale: ${{ steps.stale.outputs.stale }} strategy: fail-fast: false # n-1/n-2 (ROCMAI-430) only fan out on a release-branch push — see the @@ -253,7 +268,18 @@ jobs: && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu')) ) continue-on-error: false + # Job-level, so it replaces the workflow's `contents: read`: that is + # restated. `pull-requests: read` is for the stale-run guard's PR lookup. + # No write scope: this job builds and runs PR code on a persistent runner, + # and untrusted code and a write-scoped token do not share a job here (the + # cancel lives in `cancel-stale-run`, which runs none). + permissions: + contents: read + pull-requests: read env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Bound serve readiness below the 35-min job cap so a serve that never comes # ready fails the scenario with a real error instead of hanging until the # job is cancelled. 300s is ample for a real MI300X vLLM cold-start; @@ -265,7 +291,63 @@ jobs: # a single nightly scenario (e.g. the 27B serve) without the full nightly run. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" steps: + # Stale-run guard: a run whose PR was closed or merged, or whose head has + # moved on, has no use for the MI300X. `concurrency` cancels superseded + # runs, but a job already waiting for a self-hosted runner does not always + # leave the queue (see `pr-closed`), so the job checks the PR the moment it + # gets a runner, before the ~45 min of build + serve work, and ends without + # running anything. It only reads and records `stale`; `cancel-stale-run` + # (a hosted job, no checkout) then cancels the run. + # + # This has to be a step: a job-level `if` is evaluated before the job is + # queued, when the PR may still be open and current, and cannot be + # re-evaluated later. Every later step is gated on `steps.stale.outputs` + # so the job ends without running them, and without failing; `e2e-report` + # reads the `stale` job output so that is not reported as a lane that ran + # without uploading its report. + # + # Fails open: if the PR cannot be read (or `gh` is missing), the job runs + # as before. One assumption: any API head that differs from the event's + # counts as newer. A read that lags a fresh push could show an older head + # and wrongly end a current run (and, for a same-repo PR, cancel it); that + # costs the run's non-blocking lanes (the cancel ends the whole run), which a + # re-run recovers, so it is accepted. + - name: Skip a stale run (closed PR or superseded head) + id: stale + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if ! command -v gh >/dev/null; then + echo "::warning::gh is not installed on this runner; the stale-run guard did not run" + exit 0 + fi + if ! pr=$(gh api "repos/${GH_REPO}/pulls/${PR_NUMBER}" \ + --jq '[.state, .head.sha] | @tsv'); then + echo "::warning::could not read PR #${PR_NUMBER}; running the job unchecked" + exit 0 + fi + IFS=$'\t' read -r state head_sha <<<"$pr" + if [ -z "$state" ] || [ -z "$head_sha" ]; then + echo "::warning::PR #${PR_NUMBER} came back without a state or head sha; running the job unchecked" + exit 0 + fi + if [ "$state" = closed ]; then + reason="PR #${PR_NUMBER} is closed" + elif [ "$head_sha" != "$EVENT_HEAD_SHA" ]; then + reason="PR #${PR_NUMBER} head moved from ${EVENT_HEAD_SHA} to ${head_sha}" + else + exit 0 + fi + echo "stale=true" >> "$GITHUB_OUTPUT" + echo "::notice::${reason}; skipping this job" + echo "- :no_entry_sign: \`e2e-gpu\` skipped itself: ${reason}" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: steps.stale.outputs.stale != 'true' with: persist-credentials: false @@ -276,6 +358,7 @@ jobs: # also names an E2E root — see the exposure documented in # scripts/reclaim-gpu.sh. - name: Reclaim GPU from stray E2E processes + if: steps.stale.outputs.stale != 'true' run: bash scripts/reclaim-gpu.sh # GPU preflight: fail fast (~90s) instead of hanging to the job cap when the @@ -288,6 +371,7 @@ jobs: # floor stays for the case the process gate cannot see: a non-E2E tenant # exhausting a shared host. See .github/scripts/gpu-preflight.sh. - name: GPU preflight (bounded wait for an available GPU) + if: steps.stale.outputs.stale != 'true' env: GPU_PREFLIGHT_MIN_FREE_GIB: 16 run: | @@ -299,6 +383,7 @@ jobs: # GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes # the local target. - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 + if: steps.stale.outputs.stale != 'true' with: cache: false @@ -308,6 +393,7 @@ jobs: # the checkout's parent — untouched by git clean and persistent between jobs # on a self-hosted runner), so cargo rebuilds incrementally. - name: Run E2E tests on GPU hardware + if: steps.stale.outputs.stale != 'true' run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" # Share heavy immutable artifacts (TheRock runtimes ~3.3GB, HF weights, @@ -431,27 +517,267 @@ jobs: # literally, so a GH-expression-computed name would be invisible to it. The # `current` leg's name is UNCHANGED from before this matrix existed — it's # also what a single-element (non-release-branch) matrix always uploads. + # `e2e-gpu-selfhosted-report`, not `e2e-gpu-report`: that name is the pull + # request gate's (`e2e-gpu-queue`), whose queue run is the MI300X lane for + # pull requests; this one, on the self-hosted runners, shows beside it as + # "MI300X (self-hosted)". - name: Upload E2E report (current) - if: always() && matrix.sdk_version == 'current' + if: always() && steps.stale.outputs.stale != 'true' && matrix.sdk_version == 'current' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: e2e-gpu-report + name: e2e-gpu-selfhosted-report path: tests/e2e-cucumber/results/ - name: Upload E2E report (n-1) - if: always() && matrix.sdk_version == 'n-1' + if: always() && steps.stale.outputs.stale != 'true' && matrix.sdk_version == 'n-1' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-gpu-n-1-report path: tests/e2e-cucumber/results/ - name: Upload E2E report (n-2) - if: always() && matrix.sdk_version == 'n-2' + if: always() && steps.stale.outputs.stale != 'true' && matrix.sdk_version == 'n-2' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-gpu-n-2-report path: tests/e2e-cucumber/results/ + # Cancels a run `e2e-gpu` found stale. A separate job, on a GitHub-hosted + # runner with no checkout, so the `actions: write` token never shares a job + # with PR code (`e2e-gpu` runs it on a persistent runner). It is the internal + # half of this workflow's split: only a same-repo PR is sent a cancel. A fork + # PR (external; GitHub holds its self-hosted lanes for approval where the + # repository setting requires it, and gives it a read-only token) just has + # `e2e-gpu`'s steps skipped. Cancelling the whole run frees the PR's + # concurrency group and stops its other stale lanes; the MI300X itself was + # already freed when `e2e-gpu` skipped itself. A failed cancel (e.g. a token + # without `actions: write`) is tolerated. `e2e-report` is gated on `always()`, + # so it still runs after the cancel, but its missing-report check is gated on + # `!cancelled()` and is skipped in a cancelled run. The check's `stale` + # exemption is therefore for runs that are not cancelled: a fork PR (no + # cancel is sent), or a cancel that fails or loses a race with the check. + cancel-stale-run: + name: Cancel a stale run + needs: [e2e-gpu] + if: >- + always() + && github.event_name == 'pull_request' + && github.event.pull_request.head.repo.full_name == github.repository + && needs.e2e-gpu.outputs.stale == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Cancel this run + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + gh run cancel "$GITHUB_RUN_ID" || echo "::warning::could not cancel run ${GITHUB_RUN_ID}; its other lanes keep running" + + # The MI300X lane of a pull request, run on the shared GPU queue instead of on + # a self-hosted runner that has to be free (docs/nightly-on-the-queue.md): it + # dispatches queue-pr-run.yml on main with the PR's branch, waits for that run, + # and publishes its release-channel report as `e2e-gpu-report`, the MI300X cell + # of the consolidated grid. The queue run is the primary MI300X check; `e2e-gpu` + # above is the second. This job runs on a GitHub-hosted runner and holds no queue + # credential: dispatching needs `actions: write`, nothing more. + # + # Only a pull request from this repository: a fork's token cannot dispatch, and + # the queue only checks out branches of this repository. A run the queue does + # not take (too many going, or the nightly's window) fails this job, with the + # queue's reason, rather than passing: a missing report is never green. + e2e-gpu-queue: + name: E2E tests (MI300X, queue) + runs-on: ubuntu-24.04 # one LTS: only calls the GitHub API and copies files + # The queue run's own limit is 170 min; the dispatch and the download add + # minutes. + timeout-minutes: 180 + needs: [changes] + permissions: + actions: write + # Set when the queue refused for the nightly window: a skip, not a failure. + outputs: + skipped: ${{ steps.queue.outputs.skipped }} + # No `always()`: a job that keeps its condition true through a cancellation + # is not cancelled, and a superseded push must stop this job from polling. + if: >- + needs.changes.result == 'success' + && needs.changes.outputs.serve == 'true' + && github.event_name == 'pull_request' + && github.event.action != 'closed' + && github.event.pull_request.head.repo.full_name == github.repository + && github.actor != 'dependabot[bot]' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + QUEUE_PR_GATE: ${{ vars.QUEUE_PR_GATE }} + steps: + - name: Run on the queue + id: queue + run: | + set -euo pipefail + # The switch for the queue side: the gate does nothing until the + # repository variable QUEUE_PR_GATE is `true`, which is set once the queue + # side (the runner's allow-list and the pull request run kind) is + # deployed. Without it, a merge of queue-pr-run.yml ahead of the queue + # side would turn every pull request red. + if [ "${QUEUE_PR_GATE:-}" != true ]; then + echo "skipped=true" >> "$GITHUB_OUTPUT" + echo "::notice::the queue lane is switched off: the repository variable QUEUE_PR_GATE is not true" + echo "The queue lane is switched off until the queue side is deployed: set the repository variable QUEUE_PR_GATE to true." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + # The branch name is the PR author's: checked before it goes anywhere. + if ! [[ "$HEAD_REF" =~ ^[A-Za-z0-9_][A-Za-z0-9._/-]{0,199}$ ]]; then + echo "::error::the branch name is not one the queue takes: it must start with a letter, a digit or an underscore, hold only letters, digits and . _ / -, and be at most 200 characters" + exit 1 + fi + # A dispatch runs the workflow as it is on main. The pull request that + # adds queue-pr-run.yml cannot dispatch it until it is merged: that is a + # skip, said so, not a failure. It is a skip only when the branch itself + # carries the file; a file gone from main and from the branch (renamed or + # removed) stays red, as does any other error from either check. Drop + # this once queue-pr-run.yml is on main. + if ! out=$(gh api "repos/${GH_REPO}/contents/.github/workflows/queue-pr-run.yml?ref=main" --jq .name 2>&1); then + if [[ "$out" == *"HTTP 404"* ]] && + gh api "repos/${GH_REPO}/contents/.github/workflows/queue-pr-run.yml?ref=${HEAD_REF}" --jq .name > /dev/null 2>&1; then + echo "skipped=true" >> "$GITHUB_OUTPUT" + echo "::notice::queue-pr-run.yml is not on main yet: the queue lane cannot run" + echo "The queue lane does not run: this branch adds queue-pr-run.yml and a dispatch uses main's copy." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "::error::queue-pr-run.yml is not on main, or main could not be checked: ${out}" + exit 1 + fi + caller="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + gh workflow run queue-pr-run.yml --ref main -f "branch=${HEAD_REF}" -f "caller=${caller}" + # The run is not listed at once: the step that cancels it looks for it + # when this job ends before the id was found, but only after a dispatch. + touch "${RUNNER_TEMP}/queue-dispatched" + + # The dispatch does not say which run it started; the run's title names + # this job, so look for that. + run_id="" + for _ in $(seq 1 36); do + # A failed call is asked again, not the end of the job. + run_id=$(gh run list --workflow queue-pr-run.yml --event workflow_dispatch \ + --limit 50 --json databaseId,displayTitle | + jq -r --arg c "caller ${caller};" '[.[] | select(.displayTitle | contains($c))][0].databaseId // empty') || run_id="" + [ -n "$run_id" ] && break + sleep 5 + done + if [ -z "$run_id" ]; then + echo "::error::the dispatched queue run did not appear within 3 minutes" + exit 1 + fi + url="https://github.com/${GH_REPO}/actions/runs/${run_id}" + echo "queue run: ${url}" + # Kept for the step that cancels the run if this job is. + echo "$run_id" > "${RUNNER_TEMP}/queue-run-id" + echo "### MI300X on the queue" >> "$GITHUB_STEP_SUMMARY" + echo "Run: ${url}" >> "$GITHUB_STEP_SUMMARY" + + # Polled once a minute, not `gh run watch`, which polls every few + # seconds against this job's token. The job's own timeout bounds this, + # and a call that fails (a 502, a rate limit) is asked again. + failures=0 + while true; do + if status=$(gh run view "$run_id" --json status --jq .status); then + failures=0 + else + status="" + failures=$((failures + 1)) + # A run that cannot be read (deleted, a token problem) would + # otherwise hold this job to its timeout. + if [ "$failures" -ge 10 ]; then + echo "::error::could not read the queue run's status ${failures} times in a row" + exit 1 + fi + fi + [ "$status" = completed ] && break + sleep 60 + done + # The run has finished: a failed call is asked again, a few times and + # a while apart, so two API errors in a row do not turn a passing run red. + conclusion=unknown + for attempt in 1 2 3 4 5; do + conclusion=$(gh run view "$run_id" --json conclusion --jq .conclusion) && [ -n "$conclusion" ] && break + conclusion=unknown + sleep $((attempt * 10)) + done + echo "queue run concluded: ${conclusion}" + + # What the run published: the lanes' files, one directory per lane, + # each with a directory per channel. Only the MI300X lane's release + # channel is the report of this job. + mkdir -p queue-results report + gh run download "$run_id" --name queue-pr-run-results --dir queue-results || + echo "::warning::the queue run published no results" + # The nightly's window is by design, not a fault: the lane is skipped + # for this push, said in the summary, and the job ends green with no + # report, which the consolidated report reads as skipped. Any other + # refusal stays red. + if [ -f queue-results/refused.txt ] && [[ "$(head -n 1 queue-results/refused.txt)" == nightly:* ]]; then + echo "skipped=true" >> "$GITHUB_OUTPUT" + echo "::notice::the queue lane does not run for this push: the nightly window" + echo "The queue lane will not run for this push: the queue side keeps its GPUs for the nightly while the nightly's window lasts. Re-run this job once the window has passed." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if [ -f queue-results/refused.txt ]; then + echo "::error::the queue did not take the run: $(head -n 1 queue-results/refused.txt)" + echo "The queue did not take the run: $(head -n 1 queue-results/refused.txt)" >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + results=queue-results/mi300x/release/results + if [ -f "${results}/report.json" ]; then + cp -R "${results}/." report/ + else + echo "::error::the queue run left no MI300X release report (it ${conclusion}); it may be incomplete, which is capacity, not a failing test" + echo "No MI300X release report: the run ${conclusion}. Incomplete lanes are capacity, not a failing test; see the run above." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + # A failing suite fails the queue run, and the report above shows + # which scenarios; the job follows the run's verdict. + [ "$conclusion" = success ] + + # A push that supersedes this one cancels the job, and a job that fails + # after the dispatch (the run never appeared in the list) ends too; either + # would otherwise leave its queue run holding the runner it waits on: cancel + # that run too. Cancelling a run that has finished is harmless. The queue + # side's lanes are left to their own budgets. + - name: Cancel the queue run this job started + if: failure() || cancelled() + run: | + run_id="" + [ -f "${RUNNER_TEMP}/queue-run-id" ] && run_id=$(cat "${RUNNER_TEMP}/queue-run-id") + # The run was dispatched but its id was not found yet: look for it by the + # same title, as long as the main step would have (a run is not listed + # at once). Nothing was dispatched without the marker: nothing to look for. + if [ -z "$run_id" ] && [ -f "${RUNNER_TEMP}/queue-dispatched" ]; then + caller="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + for _ in $(seq 1 36); do + run_id=$(gh run list --workflow queue-pr-run.yml --event workflow_dispatch \ + --limit 50 --json databaseId,displayTitle | + jq -r --arg c "caller ${caller};" '[.[] | select(.displayTitle | contains($c))][0].databaseId // empty') || run_id="" + [ -n "$run_id" ] && break + sleep 5 + done + fi + [ -n "$run_id" ] || exit 0 + gh run cancel "$run_id" || true + + # A report is uploaded whatever the verdict, as the self-hosted lane's is: + # a failing run's report is what the consolidated grid is for. + - name: Upload E2E report + if: always() && hashFiles('report/report.json') != '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-gpu-report + path: report/ + # Second AMD GPU architecture: the Strix Halo (gfx1151) Ubuntu lane. DevLab # Dispatch ephemeral pool, opt-in only via the `devlab-dispatch` label (see # e2e-gpu for why an unscoped label can silently mis-route a job to the wrong @@ -478,8 +804,9 @@ jobs: runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. - # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only - # lane where vLLM runs) and one Windows pool (Strix Halo). Not on + # Not on pull_request: a PR gets one Linux GPU silicon (MI300X, the only + # silicon where vLLM runs, as `e2e-gpu-queue` and `e2e-gpu`) and one Windows + # pool (Strix Halo). Not on # merge_group either: over 7 days 35% of its merge-queue jobs (98 of 281) were # cancelled at the 45 min cap, and the queue does not wait for this # non-required check. Still runs on push (main, release/**), dispatch and @@ -515,6 +842,9 @@ jobs: - channel: nightly sdk_version: n-2 env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_SERVE_TIMEOUT_SECS: "300" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on @@ -669,6 +999,9 @@ jobs: ) continue-on-error: false env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on # an idle runner can miss it. Raise the wait budget rather than let that @@ -731,8 +1064,10 @@ jobs: # available so a wedged/held GPU fails fast instead of hanging to the cap. # BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here — # if no rocm-smi is found we WARN and continue rather than false-fail a - # working runner. When rocm-smi IS present we poll free memory the same way - # as the Linux jobs and fail only after the ceiling. + # working runner; the harness's E2E_EXPECT_GPU=1 check is the backstop + # that fails the lane if `rocm examine` reports no usable GPU at all. + # When rocm-smi IS present we poll free memory the same way as the Linux + # jobs and fail only after the ceiling. # # WHAT THIS DOES NOT CATCH, and the Linux lanes now do. This is a # free-memory floor ONLY. A fixed 8 GiB against a whole ~62 GiB gfx1151 @@ -1020,6 +1355,13 @@ jobs: ) continue-on-error: false env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled ("T h e f i l e c a n n o t ...") when captured # through PowerShell, which is exactly what a genuine wsl.exe error looks @@ -1296,12 +1638,14 @@ jobs: # today: @requires-gpu scenarios already resolve to skip here, since # `rocm-smi` is absent too (see the GPU preflight below). If this lane # ever needs a real GPU, provision it through the CLI as a separate, - # explicitly-not-under-test step rather than restoring the old script. + # explicitly-not-under-test step rather than restoring the old script, + # and flip the job's E2E_EXPECT_GPU to "1". # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results — though "held" means less here # than on the native lanes: this body keeps only the free-memory floor and @@ -1323,7 +1667,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1423,7 +1767,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1480,7 +1824,7 @@ jobs: # Forward the job-level E2E_* env vars into the guest: `wsl.exe` # does not share the calling process's environment unless a # variable is named in WSLENV. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli @@ -1615,6 +1959,7 @@ jobs: needs: - changes - e2e-gpu + - e2e-gpu-queue - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows - e2e-wsl @@ -1663,7 +2008,8 @@ jobs: # the reports that arrived, so that lane would drop out of the grid with # no row and no FAIL. On a `continue-on-error` lane nothing else is red # either. So check it here: every lane that ran (any `needs` result but - # `skipped`) must have uploaded its report artifact. This reads the run's + # `skipped`, and not self-skipped as `stale`) must have uploaded its report + # artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can @@ -1685,7 +2031,8 @@ jobs: GH_TOKEN: ${{ github.token }} NEEDS: ${{ toJSON(needs) }} LANE_REPORTS: | - e2e-gpu=e2e-gpu-report + e2e-gpu=e2e-gpu-selfhosted-report + e2e-gpu-queue=e2e-gpu-report e2e-gpu-strix-ubuntu=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-strix-windows=e2e-gpu-strix-windows-report e2e-wsl=e2e-gpu-strix-wsl-report @@ -1701,6 +2048,18 @@ jobs: [ -n "$job" ] || continue result=$(jq -r --arg job "$job" '.[$job].result // "absent"' <<<"$NEEDS") [ "$result" = skipped ] && continue + # A lane that publishes `stale=true` skipped its uploads by design + # and is not a missing report. Only e2e-selfhosted.yml's `e2e-gpu` + # stale-run guard publishes it; the line is inert in nightly.yml and + # is there because a contract test keeps the two copies of this script + # identical. Say so in the summary rather than dropping the lane silently. + if [ "$(jq -r --arg job "$job" '.[$job].outputs.stale // ""' <<<"$NEEDS")" = true ]; then + echo "- :no_entry_sign: \`$job\` skipped itself as stale; it has no column in the grid above" >> "$GITHUB_STEP_SUMMARY" + continue + fi + # A lane that said it skipped itself on purpose (the queue's nightly + # window) uploads nothing and is not missing. + [ "$(jq -r --arg job "$job" '.[$job].outputs.skipped // ""' <<<"$NEEDS")" = true ] && continue IFS=',' read -ra wanted <<<"$artifacts" for artifact in "${wanted[@]}"; do if ! grep -qxF "$artifact" <<<"$uploaded"; then diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5399e19e9f..623b9f4614 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -404,7 +404,8 @@ jobs: # to stay fast. # Runs unconditionally on the schedule/dispatch (independent of the nightly # BUILD skip above — a regression run is worth doing even with no new commits). - # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. + # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job (minus its PR-only + # stale-run guard) but sets E2E_INCLUDE_NIGHTLY. e2e-gpu-nightly: name: E2E tests (MI300X, incl. nightly-only) timeout-minutes: 90 @@ -416,6 +417,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Include @nightly scenarios (the large-model serve). See src/expectation.rs. E2E_INCLUDE_NIGHTLY: "1" # Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large @@ -506,257 +510,6 @@ jobs: name: e2e-gpu-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 - # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv - # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: - # the report crate asserts the nightly lanes publish the same platforms as the - # per-PR lanes. - e2e-gpu-nightly-rad3: - name: E2E tests (rad3 R9700, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, r9700] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, a flat - # constant rather than a proportion of capacity — high enough to catch a - # leftover serve, low enough that a clean pod passes on the first poll; do - # NOT scale it down for a smaller part. On a 32GB card it still only trips - # past 16 GiB held, so the floor alone would miss a leftover serve holding - # less than that; the process gate is what catches it (#446). Same as the - # per-PR rad3 lane. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on rad3 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on the runner-work PVC, which survives the pod - # being scaled away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm` (EAI-8057). - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-rad3-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - e2e-gpu-nightly-mi350p: - name: E2E tests (MI350P, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, mi350p] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. Floor matches the per-PR MI350P lane and the other - # Instinct lane — low enough that a clean pod passes on the first poll, - # and against a pool this size it only trips near exhaustion, so the - # process gate is what actually catches a leftover serve (#446). See - # .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on MI350P (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled - # away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-mi350p-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which - # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared - # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in - # step: the report crate asserts the nightly lanes publish the same platforms - # as the lanes in e2e-selfhosted.yml. - e2e-gpu-nightly-w7900: - name: E2E tests (W7900, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, w7900] - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - continue-on-error: true - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, which is - # a flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. Same as the per-PR - # W7900 lane — do NOT scale it down for a smaller part. On a 48GB card it - # only trips past ~32 GiB held, so the process gate is what catches a - # leftover serve holding less (#446). See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on W7900 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven @@ -781,6 +534,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -868,6 +624,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -1135,6 +894,13 @@ jobs: matrix: channel: [release, nightly] env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled when captured through PowerShell. Matches the # ubuntu/WSL composite action's own env. @@ -1334,8 +1100,9 @@ jobs: # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results. # @@ -1356,7 +1123,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1456,7 +1223,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1507,7 +1274,7 @@ jobs: shell: powershell run: | # See .github/scripts/Invoke-WslBash.ps1. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli @@ -1576,7 +1343,13 @@ jobs: # a failed step there does not turn the lane's check red. Only the # WSL lanes need this guard: every other lane writes report.json # directly in-job, with no guest-to-host copy step that can fail on - # its own. + # its own. A harness preflight refusal is not one of those: + # `run_suite` in tests/e2e-cucumber/tests/e2e.rs creates report.json + # empty before it calls `lane_verdict::preflight`, and the + # `if: always()` copy-out above carries it to the host, so it uploads + # fine and the missing-report check does not catch it; its EMPTY row + # in the report is the signal (a post-run lane failure shows as a + # FAIL row with lane_failure). if-no-files-found: error # The nightly lanes each uploaded a report nobody joined up, so reading a @@ -1596,9 +1369,6 @@ jobs: actions: read needs: - e2e-gpu-nightly - - e2e-gpu-nightly-rad3 - - e2e-gpu-nightly-mi350p - - e2e-gpu-nightly-w7900 - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly @@ -1632,20 +1402,140 @@ jobs: pattern: '*-report' path: e2e-artifacts + # The shared GPU queue's nightly (docs/nightly-on-the-queue.md) runs R9700, + # MI350P, W7900 and the Instinct lanes of the queue's fleet on its own + # schedule, and its results are published as one `queue-run-results` + # artifact of that run, not as `*-report` artifacts of this one. So fetch + # the newest such artifact of a run on main that finished, and started, in + # the last 24 hours (a run still going, or one from an earlier night, is + # not today's), and the job log of that run, which is the only place the + # queue prints which branch it tested and which lanes it blocks on. The + # artifact is found by its name, not through the workflow that makes it: + # that workflow runs on a runner no other workflow may name or depend on. + # A conclusion of `failure` is no reason to skip it: a red lane fails the + # queue's run and the artifact is still uploaded. This step never fails the + # job and never stops the report: when there is nothing to import it + # records why, and the next step says so at the top of the page instead of + # leaving the queue's GPUs out without a word. + - name: Fetch the latest queue results + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + mkdir -p queue-results + absent() { + echo "$1" > queue-results/absent.txt + echo "::warning::No queue lanes in this report: $1" + exit 0 + } + repo_id=$(gh api "repos/${GITHUB_REPOSITORY}" --jq .id) || + absent "the repository could not be read" + since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) + # Newest first: a run asked to test another branch (a `queue-run.yml` + # dispatch runs main's workflow too) must not hide an older run of main. + candidates=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | + jq -r --arg since "$since" --argjson repo "$repo_id" ' + [.artifacts[] + | select(.expired == false) + | select(.workflow_run.head_branch == "main" and .workflow_run.head_repository_id == $repo) + | select(.created_at >= $since)] + | sort_by(.created_at) | reverse | .[:5][] | .workflow_run.id') || + absent "the queue results could not be listed" + [ -n "$candidates" ] || + absent "no queue run on main finished in the last 24 hours (a run still going is not imported)" + found="" + for candidate in $candidates; do + rm -rf queue-results/results queue-results/job.log + # A run that started a day ago is not tonight's. + run_info=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}" --jq '[.status, .run_started_at] | @tsv') || continue + # A finished run only, red or green: a run still going has a + # partial job log. + [ "${run_info%%$'\t'*}" = "completed" ] || continue + started="${run_info#*$'\t'}" + started_at=$(date -u -d "$started" +%s 2>/dev/null) && + [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || continue + gh run download "$candidate" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || continue + # The log names the branch the run tested and which lanes the queue + # blocks on; a run that tested another branch is skipped, not the end + # of the search. `e2e-queue-import` checks the log again. + # The run may have several jobs: scan them all for the one that + # prints the lane matrix. + jobs=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/jobs?per_page=100" --jq '.jobs[].id') || continue + : > queue-results/job.log + for job in $jobs; do + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log.part 2>/dev/null || continue + if grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log.part; then + mv queue-results/job.log.part queue-results/job.log + break + fi + done + rm -f queue-results/job.log.part + grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log || continue + found="$candidate" + echo "$started" > queue-results/started.txt + break + done + if [ -z "$found" ]; then + rm -rf queue-results/results queue-results/job.log + absent "none of the newest queue runs of the last 24 hours tested main with a readable log and results" + fi + echo "$found" > queue-results/run-id.txt + + # Turn the run's lanes into reports of this job's own kind, and write the + # status section for the top of the matrix (`cargo xtask e2e-queue-import`). + # A lane the queue marks advisory is labelled so and does not count towards + # the release-readiness verdict; a blocking lane with no result does. + - name: Add the queue lanes to the report + continue-on-error: true + run: | + set -uo pipefail + mkdir -p queue-results + if [ -s queue-results/run-id.txt ] && [ -d queue-results/results ]; then + run_id=$(cat queue-results/run-id.txt) + args=(--queue-dir queue-results/results + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + --run-started "$(cat queue-results/started.txt 2>/dev/null)") + [ -f queue-results/job.log ] && args+=(--log queue-results/job.log) + else + args=(--absent "$(cat queue-results/absent.txt 2>/dev/null || echo 'fetching the queue results did not complete')") + fi + cargo xtask e2e-queue-import "${args[@]}" \ + --artifacts-dir e2e-artifacts \ + --section-out queue-results/queue-lanes.md || { + # Half an import is worse than none, and none is said at the top of the page. + rm -rf e2e-artifacts/e2e-queue-* queue-results/queue-lanes.md + echo "::warning::The queue lanes could not be added to the report." + cargo xtask e2e-queue-import --absent "the queue lanes could not be imported" \ + --artifacts-dir e2e-artifacts --section-out queue-results/queue-lanes.md || true + } + + # The queue lanes' status leads the page. The matrix keeps its own first + # table, the one `release-readiness` reads its verdict from. - name: Build consolidated report + step summary run: | mkdir -p consolidated cargo xtask e2e-report \ --artifacts-dir e2e-artifacts \ --html-out consolidated/index.html > consolidated/support-matrix.md + if [ -f queue-results/queue-lanes.md ]; then + cat queue-results/queue-lanes.md consolidated/support-matrix.md > "$RUNNER_TEMP/with-queue.md" + mv "$RUNNER_TEMP/with-queue.md" consolidated/support-matrix.md + fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" - # A lane that dies before writing report.json (preflight, build, a crash - # before results exist) uploads nothing, and `discover()` can only render - # the reports that arrived, so that lane would drop out of the grid with - # no row and no FAIL. On a `continue-on-error` lane nothing else is red - # either. So check it here: every lane that ran (any `needs` result but - # `skipped`) must have uploaded its report artifact. This reads the run's + # A lane that dies before writing report.json (the GPU preflight step, a + # build failure, a crash before results exist) uploads nothing. The + # harness's own preflight refusal is different: `run_suite` in + # tests/e2e-cucumber/tests/e2e.rs creates report.json empty before it + # calls `lane_verdict::preflight`, and each lane's `if: always()` upload + # step publishes it, so it shows as an EMPTY row (a post-run lane failure + # as a FAIL row with lane_failure), not as missing. `discover()` can only + # render the reports that arrived, so a lane that uploads nothing would + # drop out of the grid with no row and no FAIL. On a `continue-on-error` + # lane nothing else is red either. So check it here: every lane that ran + # (any `needs` result but `skipped`, and not self-skipped as `stale`) + # must have uploaded its report artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can @@ -1667,9 +1557,6 @@ jobs: NEEDS: ${{ toJSON(needs) }} LANE_REPORTS: | e2e-gpu-nightly=e2e-gpu-release-report,e2e-gpu-nightly-report - e2e-gpu-nightly-rad3=e2e-gpu-rad3-release-report,e2e-gpu-rad3-nightly-report - e2e-gpu-nightly-mi350p=e2e-gpu-mi350p-release-report,e2e-gpu-mi350p-nightly-report - e2e-gpu-nightly-w7900=e2e-gpu-w7900-release-report,e2e-gpu-w7900-nightly-report e2e-gpu-nightly-strix=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-nightly-strix-windows=e2e-gpu-strix-windows-release-report,e2e-gpu-strix-windows-nightly-report e2e-wsl-nightly=e2e-gpu-strix-wsl-release-report,e2e-gpu-strix-wsl-nightly-report @@ -1685,6 +1572,18 @@ jobs: [ -n "$job" ] || continue result=$(jq -r --arg job "$job" '.[$job].result // "absent"' <<<"$NEEDS") [ "$result" = skipped ] && continue + # A lane that publishes `stale=true` skipped its uploads by design + # and is not a missing report. Only e2e-selfhosted.yml's `e2e-gpu` + # stale-run guard publishes it; the line is inert in nightly.yml and + # is there because a contract test keeps the two copies of this script + # identical. Say so in the summary rather than dropping the lane silently. + if [ "$(jq -r --arg job "$job" '.[$job].outputs.stale // ""' <<<"$NEEDS")" = true ]; then + echo "- :no_entry_sign: \`$job\` skipped itself as stale; it has no column in the grid above" >> "$GITHUB_STEP_SUMMARY" + continue + fi + # A lane that said it skipped itself on purpose (the queue's nightly + # window) uploads nothing and is not missing. + [ "$(jq -r --arg job "$job" '.[$job].outputs.skipped // ""' <<<"$NEEDS")" = true ] && continue IFS=',' read -ra wanted <<<"$artifacts" for artifact in "${wanted[@]}"; do if ! grep -qxF "$artifact" <<<"$uploaded"; then diff --git a/.github/workflows/queue-pr-run.yml b/.github/workflows/queue-pr-run.yml new file mode 100644 index 0000000000..17a9535e55 --- /dev/null +++ b/.github/workflows/queue-pr-run.yml @@ -0,0 +1,75 @@ +# GPU E2E on the shared queue for one pull request's branch — start a run, wait +# for it, publish it. +# +# Dispatched by a pull request's gate job in e2e-selfhosted.yml, on main, with the +# branch to test. It is queue-run.yml with the pull request preset: the queue side +# runs one GPU model on the release channel, with a shorter wait and run time than +# the nightly, a limited number at once (the queue side sets the limit), and it +# does not take one while the nightly goes (it refuses, and this job then fails with that reason in refused.txt). +# Like queue-run.yml it needs no credential and no permission, and the runner +# runs only the allow-listed workflows as they are on main; that is enforced on +# the runner, outside this repository. +# What that protects is in docs/nightly-on-the-queue.md. +name: E2E on the queue (pull request) + +# The run's title names the job that asked for it, which is how that job finds +# this run among the dispatched ones; the caller is only ever a title here. +run-name: Pull request run (caller ${{ inputs.caller }}; branch ${{ inputs.branch }}) + +on: + workflow_dispatch: + inputs: + branch: + description: "The branch of this repository to test" + type: string + required: true + caller: + description: "The run that asked for this one, for its title" + type: string + required: true + +# Nothing here reads the repository. Uploading files uses the per-job runtime +# token GitHub issues to every job regardless of these permissions. +permissions: {} + +# One at a time per branch: a run that has started is never cancelled for a +# newer one, but GitHub still cancels an older run that is only pending when a +# newer one joins the group, and the gate reports that run as having left no +# report. The queue side limits how many go at once. +concurrency: + group: queue-pr-run-${{ inputs.branch }} + cancel-in-progress: false + +jobs: + run: + name: Run and publish + runs-on: [queue-report] + # A lane waits at most 45 min to be started, then runs for at most 75 min + # counted from its start. Finishing and publishing the run can take 30 min + # more, and 20 min are margin for the runner's own delays: 170 min. + timeout-minutes: 170 + steps: + # Anyone who can dispatch names the branch, not only the gate job, so it is + # checked here, on main, and not only in the gate: a plain branch name that + # cannot start with a dash. + - name: Check the branch name + env: + BRANCH: ${{ inputs.branch }} + # One line: the workflow contract forbids backslashes in this file. + run: if [[ "${BRANCH}" =~ ^[A-Za-z0-9_][A-Za-z0-9._/-]{0,199}$ ]]; then exit 0; fi; echo "::error::not a branch name"; exit 1 + + - name: Run on the queue and publish + env: + BRANCH: ${{ inputs.branch }} + # One line: the workflow contract forbids backslashes in this file. + run: /opt/queue/bin/queue-publish --out "${RUNNER_TEMP}/results" --start "${BRANCH}" --kind pr + + - name: Upload lane results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: queue-pr-run-results + path: ${{ runner.temp }}/results + # The publisher's files only; a lane's hidden files are not published. + include-hidden-files: false + if-no-files-found: warn diff --git a/.github/workflows/queue-run.yml b/.github/workflows/queue-run.yml index deec8fb115..e1a083798b 100644 --- a/.github/workflows/queue-run.yml +++ b/.github/workflows/queue-run.yml @@ -7,8 +7,8 @@ # prints the run's report and lane matrix, uploads the lanes' files, and passes # or fails with the run's verdict. It needs no credential and no permission: # the runner asks from where it stands. The runner is set up to run only this -# workflow and nightly-queue.yml as they are on main; that is enforced on the -# runner, outside this repository. +# workflow, nightly-queue.yml and queue-pr-run.yml as they are on main; that is +# enforced on the runner, outside this repository. # What that protects is in docs/nightly-on-the-queue.md. name: E2E on the queue diff --git a/.github/workflows/vllm-route-check.yml b/.github/workflows/vllm-route-check.yml new file mode 100644 index 0000000000..0f937fd094 --- /dev/null +++ b/.github/workflows/vllm-route-check.yml @@ -0,0 +1,88 @@ +name: vLLM route check + +# Advisory, and deliberately NOT a required check. Resolves the SDK an unpinned +# `rocm install sdk` picks on the release and the nightly channel, and warns +# when the vLLM engine has no install route for it (see +# scripts/vllm_route_check.py). It reads the live indexes, so it can warn on a +# PR that changed nothing: an index moved on to an SDK vLLM has no wheels for. +# Required, that would block unrelated merges, so a missing route only reports. +# +# Runs on a PR only when it touches what decides the answer: the resolver that +# picks the default SDK (therock.rs and therock/), the engine's two build +# tables, or this check. CPU only: a dry run, no install. +# +# A finding is reported as a warning plus a job summary and the job stays green +# (the step swallows exit code 1), so a known gap does not leave a permanent red +# X on every PR touching these paths. The build, the self-test and a check that +# cannot run (exit 2) still fail it. + +on: + pull_request: + paths: + - apps/rocm/src/therock.rs + - apps/rocm/src/therock/** + - engines/vllm/src/wheel_builds.rs + - engines/vllm/src/discover_builds.rs + - scripts/vllm_route_check.py + - .github/workflows/vllm-route-check.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: -D warnings + AWS_LC_SYS_USE_SYSTEM: "0" + +jobs: + vllm-route: + name: vLLM route for the default SDKs (advisory) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 + with: + cache-key: ubuntu-24.04 + + - name: Build rocm + run: cargo build --release -p rocm + + - name: Self-test + run: python3 scripts/vllm_route_check.py --self-test + + - name: Check the vLLM route of the default SDKs + id: route + # Exit 1 = a default SDK has no route: reported below, job stays green. + # Any other non-zero exit (2 = the check itself could not run, for + # example the build tables no longer parse, or `rocm install sdk + # --dry-run` timed out, failed or printed no plan) fails the job. + run: | + rc=0 + python3 scripts/vllm_route_check.py --rocm target/release/rocm \ + > "$RUNNER_TEMP/route.txt" 2>&1 || rc=$? + cat "$RUNNER_TEMP/route.txt" + echo "rc=$rc" >> "$GITHUB_OUTPUT" + if [ "$rc" -gt 1 ]; then exit "$rc"; fi + + - name: Report a missing route + if: steps.route.outputs.rc == '1' + run: | + echo "::warning title=vLLM route::A default SDK has no vLLM install route. See the job summary." + { + echo "### vLLM route check: no route for a default SDK" + echo + echo "Advisory: add the discovery row once the wheels are published." + echo + echo '```' + cat "$RUNNER_TEMP/route.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43120b797d..007d1df18d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: args: [--fix=lf] # repo standardises on LF endings (see .gitattributes) exclude: '^THIRD_PARTY_NOTICES\.txt$' - # Python: lint + format the helper scripts and engine workers. + # Python: lint + format every Python file in the repository. - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.17.3 hooks: diff --git a/AGENTS.md b/AGENTS.md index 6b972da2dc..158f9c0af1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,7 +101,8 @@ definitions when no existing scenario already covers it: - if a scenario for the behavior already exists, say in the PR text which `@id:` it is and that the change makes it pass — do not silently rely on it - if the scenario can only run on a gated lane (`@requires-gpu`, `@nightly`), say so in - the PR text and name the lane that will exercise it + the PR text and name the lane that will exercise it; moving an existing scenario off the + blocking mock `e2e` job onto such a lane is justified as §8 says - purely internal changes (refactors, CI plumbing, docs) do not need one; say why in the PR text rather than leaving it unexplained @@ -130,6 +131,21 @@ derived from the same value the branch is taken on — as `preapproved_install_l `apps/rocm/src/therock/consent.rs` derives it from the approval source — prefer that: a message computed from the decision cannot disagree with it. +**A comment or doc line a change adds or edits that states how the code behaves is checked +against that code, and one that is the reason for a decision also names where the code +makes it true.** A decision such as narrowing a test's scope, documenting a limitation, +choosing a design, or dropping a guard often rests on one sentence: a reviewer who accepts +the sentence accepts the decision, and no test reads a comment. Before pushing, open the +code each claim describes and confirm it. Where the claim is the reason for a decision, +also name the place as a file and a function, constant or test +([runner.rs](crates/rocm-dash-daemon/src/runner.rs)'s `cycle_timestamp`), never as a line +number, which goes stale with the next edit above it. Where the decision rests on timing +or ordering, the claim names where the mechanism that enforces it is defined, not only +that one exists. State an invariant rather than a count: "every engine crate" survives a +new engine, where "the three engine crates" silently stops being true. When a review +finding makes you rewrite such a sentence, check the replacement the same way: a narrower +claim is harder to doubt, not more likely to be right. + **A new test is shown to fail when the code it covers is broken.** Before pushing, break the branch under test (revert the guard, swap `max` for `sum`, replace `get(i)` with `get(0)`) and confirm the test goes red. A test that stays green with its target removed @@ -194,6 +210,17 @@ Understand existing patterns first: - sibling implementations in `apps/`, `crates/`, and `engines/` - existing tests and conventions in `docs/testing.md` +Read the project's goals in the [Manifesto](README.md#manifesto) section of +`README.md` before planning a change. Where the approach a request or ticket +asks for would work against one of them (a manual workaround step, a build +installed for a GPU it was not built for), raise the conflict with the user +before implementing it. Where nobody can be asked, implement nothing that +conflicts, do the rest only where it stands on its own, and name the conflict +in the final report. The request states one need, and the goals are what the +project has decided to trade other needs against. The goals only govern +choices this file leaves open; every rule elsewhere in it, such as §6's ban on +silent CPU fallback, still binds whatever the user answers. + Fix at the correct layer (root cause), not by shrinking symptom visibility. If approach choice is ambiguous, present alternatives and recommend one. @@ -237,6 +264,14 @@ Keep docs and behavior claims in sync while editing: every review-fix commit: it lists every user-visible change (including ones on other platforms), names the lane that exercises the new behavior, and states the reason when there is no scenario +- hold the branch's commit messages to the same bar: pull requests here are normally + squash-merged with the branch's commit messages as the default squash body (rebase-merge + is also enabled; either way each commit message lands in public history). Each + describes what its own commit does, in words still true at the branch head. A message + whose words a later commit made misleading — a cross-reference to a section, item or + name that a later commit changed — is reworded before merge; a commit whose whole + subject a later commit removed is folded into that commit or dropped instead. Either + rewrites pushed commits, which needs a force-push, which needs approval (§1) ## 6) rocm-cli Architecture Guardrails @@ -407,6 +442,18 @@ coverage floor in `coverage-floors.toml`, and `cargo xtask coverage` fails when below it. If a drop is legitimate, re-bless with `cargo xtask coverage --bless` and say in the commit why — do not re-bless to clear a red check. +Moving a scenario off the blocking mock `e2e` job is a deletion too, in any change +including CI plumbing, and no coverage floor catches it. A tag that makes an existing +scenario skip there (`@nightly`, `@requires-gpu`; the tag table in +[the e2e README](tests/e2e-cucumber/README.md) says what +each tag resolves to) removes its pre-merge coverage as surely as deleting it, though the +diff shows only a tag. Justify it as a deletion: the PR text names, for each direction the +behavior could be broken in (for a flag, hardcoded on and hardcoded off), which blocking +check still fails. A scenario that is the only pre-merge guard for a seam no unit test +reaches says so in a comment beside it, as +`@id:therock-next-09-wheel-devel-adds-the-toolchain` does; when such a scenario slows or +destabilizes a neighbor, fix the neighbor at its own layer instead of moving the guard. + When relevant to touched behavior, also run targeted checks from `docs/testing.md`, such as: - focused Rust test groups for touched modules @@ -452,6 +499,17 @@ If a vendored upstream tree is introduced in the future, apply the following rul - before resuming review-feedback work on a PR you authored, first check for review comments with no reply yet (same `in_reply_to_id` cross-reference technique below) - automated reviewers (e.g. Copilot) can post new findings on a commit that itself fixed earlier findings; after pushing a fix and replying to the original threads, re-fetch PR comments once more before treating the review round as closed - to check whether a review comment already has a reply, do not call `gh api repos/OWNER/REPO/pulls/comments/$id/replies` (GET); it 404s. Fetch the full list (`gh api repos/OWNER/REPO/pulls/{pr}/comments --paginate`) and cross-reference each comment's `in_reply_to_id` against other comments' `id`s +- when reviewing a revision that answers review feedback, diff it whole against the revision you last reviewed, the same way the author does (**Review-fix commits** below), not only the lines your finding named: a remediation is written in a narrow window, gets the least scrutiny, and is where new defects arrive +- when reviewing, treat a comment or doc line in the diff that states how the code behaves as a claim to check against that code (§3), not as context; one that is the reason for a decision (§3 says which) is checked before you approve, including that it names where the code makes it true +- when reviewing, before marking an objection blocking because it rests on a timing, ordering, concurrency or "this cannot happen" argument, name the mechanism it assumes and where that is defined (file and symbol); an objection whose mechanism you cannot locate is posted as a question, not a blocking finding +- when reviewing, read the commit messages as well as the diff: each lands in public history on `main`, in the squash commit's body or, under rebase-merge, as a commit of its own (§5) + +**Review-fix commits** — a commit answering review feedback is reviewed by its author as a whole change: + +- diff it against the head the reviewer saw (the `commit.oid` of the review it answers, from `gh pr view --json reviews`, §1), not only the lines the finding named, and say in the reply whether it changes behavior outside the finding +- once the base has moved into the branch since the review, by a rebase or by merging the base in, `git diff ..HEAD` also shows everything the base gained. Fetch the PR's base branch (for a stacked PR, the branch it depends on, not `main`) and diff with `git range-diff origin/ HEAD` instead, reading in full each commit it marks new or changed. A commit it marks removed is either the base's old history, where the base was force-pushed, or a commit dropped from the branch since the review, so check which before setting one aside. `git range-diff` skips merge commits, so read a merge that resolved conflicts with `git show ` +- once a force-push has taken the reviewed head off the branch, a clone that never had it fetches it by its full SHA first, while the host still retains it (`git fetch origin `) +- a review-fix commit does not widen what a documented user-facing option means or authorizes (a `--yes` that starts granting a second consent); where widening is needed, it is its own commit, described as such in the PR body, that changes the option's help text and docs together with its behavior **Stacked and dependent PRs:** @@ -472,8 +530,9 @@ Watch checks to completion and drive to all-green. A red check means "not ready" until resolved. -After each review-fix commit, rerun the §8 gate locally and self-review against §3 to §6 before pushing. Round-trips spent on a scenario number out -of sequence or a clippy lint cost more than running them first. +After each review-fix commit, rerun the §8 gate locally and self-review against §3 to §6 +and §11's review-fix commit rules before pushing. Round-trips spent on a scenario number +out of sequence or a clippy lint cost more than running them first. ## 13) Upstream Pre-flight Checklist @@ -495,6 +554,10 @@ Run this checklist before any upstream push, PR update, issue update, or public 14. New tests shown to fail when their target is broken; state matrix written for new state or placeholders; platform-gated logic covered by a test that runs on its gate — *§3* 15. Old claims grepped across docs and PR body; PR body matches the final diff — *§5* 16. Subprocess/FFI checklist applied to any new process or driver call — *§6* +17. Comments and doc lines this change adds or edits that state how the code behaves checked against that code; each that is the reason for a decision names where the code makes it true — *§3* +18. Review-fix commits diffed whole against the reviewed head; no documented option widened inside one — *§11* +19. Any existing scenario moved off the blocking mock `e2e` job justified as a deletion — *§8* +20. Every commit message describes its own commit in words still true at the branch head — *§5* ## 14) Related Internal Workflows diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3354c2bad5..38c8895ff2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,10 @@ git config --global commit.gpgsign true See [Commit signatures](https://github.com/ROCm/rocm-cli/blob/main/docs/commit-signatures.md) for GPG signing, GitHub "Verified" status, and troubleshooting. +### Commit messages are public history + +Pull requests are normally squash-merged, and by default the squash commit's message is built from the messages of the branch's commits; rebase-merge is also enabled, and lands each commit with its own message. Either way, once merged the messages are public history that is not rewritten, so commit messages are reviewed for accuracy on the same footing as the diff: each describes what its own commit does in words still true at the branch head, and one whose words a later commit on the branch made misleading is corrected before the pull request merges. AGENTS.md section 5 states the rule for contributors and section 11 for reviewers. + ### What reviewers look for Reviewers look for the following: @@ -163,6 +167,10 @@ Reviewers look for the following: - No secrets, credentials, or internal hostnames in committed files - Third-party dependencies recorded in `Cargo.lock` - License headers on new source files (see `licenserc.toml`) +- Comments and documentation a pull request adds or changes that state how the code behaves are checked against that code, and one that is the reason for a decision names the file and the function, constant or test that makes it true, as AGENTS.md section 3 states +- A blocking objection resting on a timing, ordering, concurrency or "this cannot happen" argument names the mechanism it assumes and where that mechanism is defined, and one whose mechanism cannot be located is raised as a question instead, as AGENTS.md section 11 states +- A revision answering review feedback is re-reviewed as a whole change, and a commit in it that answers the feedback widens no documented user-facing option's meaning or authority, as AGENTS.md section 11 states +- An existing end-to-end scenario moved off the blocking mock `e2e` job (for example by a tag) is reviewed like deleting the scenario, as AGENTS.md section 8 states ## Reporting security issues diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf0..785e347897 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,6 +1282,7 @@ dependencies = [ "rand 0.10.3", "reqwest 0.13.4", "rocm-core", + "rocm-engine-protocol", "serde", "serde_json", "sha2", @@ -3888,6 +3889,7 @@ dependencies = [ "clap", "flate2", "libc", + "object", "proptest", "rocm-core", "rocm-deps", diff --git a/MANIFEST.md b/MANIFEST.md index ca33e22907..09bc5e93d1 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -714,11 +714,16 @@ Lemonade embeddable archive from the official Lemonade SDK GitHub releases at to the rocm-cli managed data directory and is reused for subsequent `rocm engines start lemonade` invocations. This is a self-contained binary distribution; no Python packages are installed for Lemonade by rocm-cli. -On Linux/WSL, `rocm engines install lemonade` may also query -`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases/latest` (an -unauthenticated call) while aligning Lemonade's `llamacpp:rocm` backend to -the active ROCm SDK version; see +On Linux/WSL and Windows, `rocm engines install lemonade` may also list +`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases` (the 30 newest +releases, one unauthenticated call) while aligning Lemonade's `llamacpp:rocm` +backend to the active ROCm SDK version; see [Lemonade backend alignment on engine install](docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install). +Lemonade itself then downloads the llama.cpp backend it installs from the +`lemonade-sdk/llama.cpp` GitHub releases, and the ROCm runtime that backend +runs on from AMD's ROCm package repository (as Python wheels into a virtual +environment of Lemonade's own, or as a TheRock tarball); those downloads are +Lemonade's, not rocm-cli's. ### Engine-Specific Python Dependencies diff --git a/PSScriptAnalyzerSettings.psd1 b/PSScriptAnalyzerSettings.psd1 index cf70d8e8b4..d4f6cda1aa 100644 --- a/PSScriptAnalyzerSettings.psd1 +++ b/PSScriptAnalyzerSettings.psd1 @@ -1,11 +1,21 @@ -# PSScriptAnalyzer configuration for the repo's PowerShell scripts -# (install.ps1, scripts/*.ps1). Used by the prek hook and CI. +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +# PSScriptAnalyzer configuration for the repo's PowerShell scripts: every tracked +# .ps1 file, as listed by `git ls-files '*.ps1'` in xtask/src/powershell.rs +# (today install.ps1 and .github/scripts/Invoke-WslBash.ps1). Used by the prek +# hook and CI. +# +# Only .ps1 files are linted. The licence-header check (licenserc.toml) also +# covers .psm1 and .psd1, so a module or data file added later gets a header but +# no analyzer pass until the xtask and the hook's `files:` pattern are extended. @{ # Enforce the full default rule set at Warning and Error severity. Severity = @('Error', 'Warning') - # install.ps1 and the acceptance/packaging scripts are user-facing console - # programs where `Write-Host` is the intended way to print progress to the - # terminal, so the "avoid Write-Host" rule does not apply here. + # install.ps1 is a user-facing console program where `Write-Host` is the + # intended way to print progress to the terminal, so the "avoid Write-Host" + # rule does not apply here. ExcludeRules = @('PSAvoidUsingWriteHost') } diff --git a/README.md b/README.md index d7461dfee1..121caaa58f 100644 --- a/README.md +++ b/README.md @@ -45,19 +45,78 @@ instances need the `rocmdashd` unix socket, so they're Linux/WSL2 only (see serving is Linux or WSL2 only (see [docs/vllm.md](docs/vllm.md)). The minimum supported Linux release, native or under WSL2, is Ubuntu 24.04. On -other distributions, the equivalent requirement is glibc 2.38 with -`GLIBCXX_3.4.32`. - -The Lemonade engine is linked against these versions, and every published build -of it needs them, so there is no older release to fall back to. Ubuntu 22.04 -ships glibc 2.35 and cannot run the engine. Ubuntu 24.04 provides glibc 2.39 and -`GLIBCXX_3.4.33`. +other distributions the equivalent requirement is glibc 2.38 with +`GLIBCXX_3.4.32`, and OpenSSL 3.0.0 or newer (`libssl.so.3`): that is what the +Lemonade engine is linked against. Every stable Lemonade release from 11.5.1 +through 2026.44.1, the one rocm-cli pins, needs those versions; the newest, +2026.45.3, needs glibc 2.43 and OpenSSL 3.2.0, which is why it is not pinned. +Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides glibc +2.39, `GLIBCXX_3.4.33` and OpenSSL 3.0.13. On an older host, +`rocm engines install lemonade` stops before downloading anything and says what +falls short: the glibc version Lemonade needs and the newest the host's +`libc.so.6` provides, or that +the host has no `libssl.so.3`, the library of OpenSSL 3. That check reads glibc +and OpenSSL only; `GLIBCXX_3.4.32` (libstdc++) is not checked. Where Lemonade is +already installed, the same check refuses to reinstall it or start it, and says +the installed copy was left as it is. The check reads the `ldconfig` cache, so a +host that provides these libraries some other way (`LD_LIBRARY_PATH`, a Nix or +Guix profile) can set `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`: the +shortfall is then printed as a warning and Lemonade goes ahead. > [!IMPORTANT] > **Tech Preview:** This software is provided as-is, without warranty or > guarantee of stability. APIs, commands, and behavior might change without > notice. Intended for experimentation and early feedback only. +## Manifesto + +ROCm CLI exists to make an AMD GPU the easiest place to run AI, not the +hardest. These are the goals we build toward. As a Tech Preview the tool does +not meet all of them yet; where it falls short, that is a bug to fix, not a +different goal. + +1. **From bare metal to inference in minutes.** A fresh machine with an AMD + GPU should get to a served model in a few commands and a few minutes (see + [Driver installation](#driver-installation) and + [Configure ROCm and serve a model](#configure-rocm-and-serve-a-model)). A + step that needs a manual workaround is a gap in the tool, not homework for + the user. +2. **Every AMD GPU, with an honest answer.** We aim to support the whole AMD + line: Instinct accelerators in the datacenter, Radeon cards on the desk, + and Ryzen AI APUs such as Strix Halo in laptops and mini-PCs, on Linux, + Windows, and WSL2. Where we know a GPU has no ROCm build, we say so rather + than install one meant for a different GPU. +3. **Rock solid.** Correctness comes before features. We test on real AMD + hardware and review new user-visible behavior against an executable + scenario. Releases are checksummed and signed, and the install scripts + verify what they download. A failure partway through should never leave + your system half-changed. +4. **The GPU or nothing.** There is no CPU fallback. If a model cannot run + on your GPU, serving fails rather than quietly running it somewhere + slower. Use `rocm diagnose --model` to check before you download. +5. **Your machine, your call.** Nothing you have should disappear without + your say-so. `rocm uninstall`, `rocm runtimes uninstall` and the + `rocm storage` removals show the plan and ask first, and `--yes` gives that + approval in advance for scripts. +6. **When it breaks, it tells you how to fix it.** `rocm diagnose` and + `rocm fix` match known failures against a catalog with a stable id for + each. Where a fix is safe to automate, `rocm fix` applies it after asking. + Otherwise it prints the steps to take, or says plainly that no fix exists. +7. **Built for people, scripts, and agents alike.** A launcher and a live + dashboard for people, `--json` output from `examine`, `diagnose`, `fix` + and other inspection commands for scripts, and the + [`rocm-doctor` skill](https://github.com/amd/skills/tree/main/skills/rocm-doctor) + so AI agents can diagnose and repair a setup with the same `rocm diagnose` + and `rocm fix` commands a person would. +8. **Open by default.** MIT-licensed and developed in the open, with its + documentation kept in the same repository as the code it describes. +9. **Extensible, from local to remote.** ROCm CLI should grow with the + open-source ecosystem, adopting new inference engines, ROCm-based + libraries, and tooling for safe, scalable agentic AI (such as agent + sandboxing and fleet orchestration) as supported, validated extensions. It + should also take you from your own machine to remote GPU compute without + switching tools; `rocm remote` (preview) is the first step there. + ## Demos ### ROCm CLI @@ -82,7 +141,8 @@ add a demo. Until the demo-gifs workflow has run once, these links 404. ## Installation -The installer downloads a prebuilt bundle, verifies its SHA-256 checksum, +The installer downloads a prebuilt bundle, verifies its SHA-256 checksum and +its detached signature against the release public key pinned in the installer, installs the `rocm` and `rocmd` binaries into `~/.local/bin`, and adds that directory to your shell `PATH`. On Linux, when fish is installed (`fish` is on `PATH`, or fish's config folder already exists), the installer also adds that @@ -409,6 +469,14 @@ the nightly channel moved on 2026-08-23. If ROCm CLI can't tell which exact GPU architecture this computer has, or you want a different one, also add `--family` with the raw `gfx` code from `rocm examine`. For example: `--family gfx1200`. +On WSL, wheel runtime installs and updates (`rocm install sdk`, +`rocm update --apply`) also link `librocdxg.so` to the runtime's own +`librocdxg.so.1`, so the runtime searches its own copy rather than an older host +`librocdxg` (it also replaces a dangling `librocdxg.so`); for a runtime +installed earlier, `rocm examine` prints a `wsl_runtime_librocdxg` line with the +command that does the same (see +[WSL](https://github.com/ROCm/rocm-cli/blob/main/docs/wsl.md)). + Running `rocm install sdk` when a managed runtime is already the active default asks first, because the new install takes over as the active default; see [ROCm installation](https://github.com/ROCm/rocm-cli/blob/main/README.md#rocm-installation) @@ -642,7 +710,13 @@ hardware, not something an install or a setting can put right. `rocm fix` takes these flags: -- `--dry-run` shows any fix's plan without changing anything. +- `--dry-run` shows a fix's plan without changing anything. On Windows, + `fix-2-unset-override`, `fix-6-path` and `fix-9-igpu-dgpu --device-index N` + read the current value of the variable they change before planning, so a dry + run whose read fails stops with exit 4 and shows no plan. It still prints the + fix's catalog entry, which describes the fix in general terms (for + `fix-9-igpu-dgpu`, a `setx HIP_VISIBLE_DEVICES ` template) and is + not a preview of the change on this machine. - `--yes` skips the interactive confirmation once you've reviewed it. Without it, a fix that changes files or settings asks `[y/N]` (default: no). When stdin is not a terminal it prints the plan and refuses with exit code 5, @@ -668,6 +742,16 @@ on how it is written: `--device-index=-1` is accepted by the parser and the fix refuses it with exit 3, while `--device-index -1` (with a space) is read as an unknown option `-1`, a usage error that exits 2. +A fix that runs a command and fails exits 4 and says why on stderr. That +includes a command that does not finish within its time limit (one minute): +the CLI stops it and exits 4, but the change may have been made in part or may +still finish, so the message says how to check it (on Windows, an expression +to run in PowerShell) and names the `rocm fix` command to re-run if it was not +made. On Windows, when one of those three fixes cannot read the current value +of the variable it changes, it stops before changing anything, also with exit 4; +under `--dry-run` it stops the same way, before showing the change it would +make. + ### ROCm installation ``` @@ -923,7 +1007,7 @@ old date is a hint to look at the driver, not a finding that it is broken. See Manage multiple side-by-side ROCm runtimes: ``` -rocm runtimes list +rocm runtimes list [--json] rocm runtimes activate [--restart-services [--yes]] rocm runtimes rollback [--restart-services [--yes]] rocm runtimes uninstall [--yes] [--dry-run] @@ -932,12 +1016,37 @@ rocm runtimes adopt --python [--root ] [--runtime-id ID] [--runtime-key KEY] [--channel LABEL] [--replace] ``` -`list` shows each runtime's version, GPU family, mode, and whether it carries -the compiler toolchain (`toolchain=included|excluded`). Runtimes are told apart -by the packages they were installed from, not by version alone, so one version -can appear more than once: a `--devel` install and a plain one of the same -version are two separate runtimes, as are two installs that resolved different -GPU device payloads. +`list` shows each runtime's version (with its build date when the version +encodes one), GPU family, mode, and whether it carries the compiler toolchain +(`toolchain=included|excluded`). A `provenance:` line under each runtime names +where it came from: its `channel`, the package `source` it was installed from +(the wheel index, or the downloaded archive for a tarball install), its +`source_layout_generation`, and the exact `device_target` its packages were +installed for. The values are the ones the runtime's manifest recorded: the +package index, else the artifact URL; the layout field, else the wheel +composition's; the recorded target, else the `device-` extra in the +package specs. For the layout and target that is the same recorded field +`rocm update` uses; the listing then trims the value and prints a blank one as +`unrecorded`, as it does a fact recorded nowhere. Nothing is guessed from other +facts (the target is never the family) — so a tarball or family-wide install +shows `device_target=unrecorded`, and an adopted runtime, which installed +nothing, shows `source=unrecorded`. An adopted runtime's channel is the one adoption +recorded: the channel `--channel` or `--runtime-id` named, `release` when neither +names one. +`rocm examine` reports the same four facts for the active runtime as +`active_runtime_channel`, `active_runtime_source`, +`active_runtime_source_layout_generation` and `active_runtime_device_target`. +Runtimes are told apart by the packages they were installed from, not by +version alone, so one version can appear more than once: a `--devel` install +and a plain one of the same version are two separate runtimes, as are two +installs that resolved different GPU device payloads. + +`list --json` prints the same listing as one JSON document on stdout for +scripts: each runtime with its key, version, mode, status and toolchain, its +`provenance` as an object with the same four facts, and `active` and +`rollback_target` flags in place of the `*` and `-` markers. Unset values, +including a provenance fact the text shows as `unrecorded`, are `null`. Fields +may be added in later releases; existing ones are not renamed or removed. `activate` and `rollback` change which runtime the next install or server picks up. A server that is already running is not moved: it keeps serving on @@ -1364,10 +1473,30 @@ the memory fraction when a small model shouldn't reserve most of a large card. `--require-api-key` makes a loopback endpoint require an API key too, which matters when something else republishes the port. A public bind generates a key -automatically unless you supply one. Set `ROCM_SERVE_API_KEY` instead of +automatically unless you supply one. The key is kept in a file under the data +directory that only your user account can read. Set `ROCM_SERVE_API_KEY` instead of passing `--api-key`, so the secret stays out of your shell history and the process table. +With the vLLM engine, an endpoint bound to a public interface +(`--allow-public-bind`), or served with `--require-api-key`, answers only +clients that present its API key as an `Authorization: Bearer` header, on every +route the server answers, not just the OpenAI `/v1` API — including vLLM's own +`/invocations`, `/tokenize` and `/metrics` — except `GET`/`HEAD /health` and +CORS preflights: the `/health` liveness check stays open. The AIM engine runs +the same vLLM server inside its container but does not install this check, so +a key-protected AIM endpoint has only vLLM's own `VLLM_API_KEY` check, which +leaves routes such as `/invocations`, `/tokenize` and `/metrics` answering +without the key. + +Because `/metrics` is covered, `rocm dash` cannot read engine metrics from a +key-protected vLLM server: it leaves that server's KV-cache, queue, throughput +and latency figures empty and keeps showing a warning such as +`vllm scrape: 1 failed (last: transport: GET http://:/metrics: status 401 Unauthorized)`. + +`rocm bench load` sends no API key, so it cannot benchmark a key-protected +endpoint on any engine. + `--temperature` (>= 0.0), `--top-p` (0.0-1.0), and `--max-tokens` (> 0) set server-wide sampling defaults for the launched engine. They apply only to `vllm` and `lemonade`; other engines reject them. For vLLM they are folded @@ -1396,7 +1525,15 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +quick smoke test (time to first token and approximate tokens per second). On an +interactive terminal, if the smoke test cannot complete, the summary is headed +`(smoke test failed)`, shows the status as `started, smoke test failed` instead +of `ready`, and a note gives the reason and points at +`rocm logs --service `. ROCm CLI does not stop the server, and the command +still exits 0. Which failures are detected, and which still read as `ready`, is +listed under +[Serve Smoke Test Failure](https://github.com/ROCm/rocm-cli/blob/main/docs/testing.md#serve-smoke-test-failure) +in the testing guide. Control returns to your shell with the server still running; manage it later with `rocm services` (below). @@ -1686,7 +1823,8 @@ bound by another process, it refuses and the service stays stopped `rocm services restart --yes` to run once that launch finishes. `remove` deletes one record that is no longer running, together with its log, -its engine state file, and its endpoint key file; a running server is refused, +its engine state file, its endpoint key file and any copy of that key an +interrupted save left behind; a running server is refused, so stop it first. `prune` does the same in bulk, always leaves running servers alone, and additionally clears leftover files whose record is already gone. A server counts as running while any process it recorded may be running, which on diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 9d57b7cb48..83567da2b2 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -19,8 +19,11 @@ e2e-test-hooks = [ ] # Test-only fault injection for the e2e suite. Enabled ONLY by the `xtask e2e` # mock-lane build (never by the release build), it compiles in a `rocm serve` -# hook that simulates a managed vLLM launch which OOM'd and never became ready, -# so the positive OOM-guidance summary can be verified on a GPU-less CI host. +# hook that simulates a managed vLLM launch, so the `rocm serve` summary can be +# verified on a GPU-less CI host: one that OOM'd and never became ready (the +# OOM guidance), or, in the hook's `ready` mode, one that became ready (the +# hook only fabricates the ready record; a mock engine supplies the failure of +# the post-launch smoke test). # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in # `service_launch/readiness.rs`. e2e-oom-fault-injection = [] @@ -75,6 +78,9 @@ zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-cry [dev-dependencies] # The fake engine child both engine IPC callers are tested against. rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } +# The Windows DACL check the tests of the owner-only key stores read their +# files back with (`owner_only_file::owner_only_dacl_mismatch`). Test-only. +rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # Property-based tests for the byte-size formatter behind the `download: # approved up to …` line. "A size is printed in its own unit" has to hold at # every unit boundary, and the narrow band where `{:.1}` rounding reaches 1024.0 diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 0b95d736b1..c2425836db 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -15,7 +15,7 @@ //! //! Callers in other modules, production and test-only, reach the moved items //! through the `chat::` module boundary, so this module re-exports each one -//! another module reads (`run_internal_mcp_call`, +//! another module reads (`run_internal_mcp_call`, `mcp_call_cli_arguments`, //! `rocm_chat_tool_system_prompt_for_host`, the tool-policy items, `any_substring` //! and `ensure_flag`, and the test-only ones under `#[cfg(test)]`). @@ -32,8 +32,13 @@ mod turn; #[cfg(all(test, unix))] pub(crate) use self::summaries::deterministic_comfyui_tool_summary; // `run_internal_mcp_call` lives in `mcp_bridge.rs` and stays reachable here as -// `chat::run_internal_mcp_call` for `main.rs` and `dash_seam.rs`. -pub(crate) use self::mcp_bridge::run_internal_mcp_call; +// `chat::run_internal_mcp_call` for `main.rs` and `dash_seam.rs`; +// `mcp_call_cli_arguments` likewise for `main.rs`'s `rocm mcp-call` dispatch. +// `internal_mcp_tool_call` is read only by `dash_seam.rs`'s tests, so its +// re-export is `#[cfg(test)]`. +#[cfg(test)] +pub(crate) use self::mcp_bridge::internal_mcp_tool_call; +pub(crate) use self::mcp_bridge::{mcp_call_cli_arguments, run_internal_mcp_call}; // `rocm_chat_tool_system_prompt_for_host` lives in `turn.rs` and stays // reachable here as `chat::rocm_chat_tool_system_prompt_for_host` for `dash.rs`. use self::turn::render_chat_prompt_result_with_progress; @@ -49,7 +54,7 @@ pub(crate) use crate::rocm_argv::ensure_flag; // here as `chat::` for `dash_seam.rs`. `ChatRocmCommandAction`, // `rocm_chat_tool_requested_args`, `chat_install_driver_action` and // `chat_rocm_command_action_from_args` are read through `chat::` only by other -// modules' tests (`driver_plan.rs`, `system_packages.rs`), and this module does +// modules' tests (`driver_plan.rs`, `system_packages.rs`, `dash_seam.rs`), and this module does // not call them itself, so their re-export below is `#[cfg(test)]`. #[cfg(test)] pub(crate) use crate::tool_policy::{ diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs index 34f3d6e052..cf5ea7c6ac 100644 --- a/apps/rocm/src/chat/mcp_bridge.rs +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -18,7 +18,12 @@ //! calls back into the turn loop in `chat/turn.rs`, tests included; the turn //! loop reads this module's items through `super::mcp_bridge`. //! `run_internal_mcp_call` stays reachable as `chat::run_internal_mcp_call` for -//! `main.rs` and `dash_seam.rs`. +//! `main.rs` and `dash_seam.rs`. It runs the argument object exactly as given, +//! through `internal_mcp_tool_call` (re-exported, test-only, for `dash_seam.rs`'s +//! tests), and never unwraps a nested `arguments` field. Unwrapping a pasted MCP +//! `tools/call` params envelope is the `rocm mcp-call` command's own boundary +//! step, `mcp_call_cli_arguments`, re-exported as `chat::mcp_call_cli_arguments` +//! for `main.rs`. use super::summaries::managed_service_running_state; use crate::automations::{render_automations_text, render_daemon_text}; @@ -142,13 +147,8 @@ pub(crate) fn run_internal_mcp_call( arguments: serde_json::Value, allow_mutation: bool, ) -> Result { - let arguments = internal_mcp_arguments(arguments); - let call = providers::ChatToolCall { - id: None, - name: name.to_owned(), - arguments: serde_json::Value::Object(arguments.clone()), - }; - validate_chat_tool_call(&call)?; + let call = internal_mcp_tool_call(name, arguments)?; + let arguments = call.arguments.as_object().cloned().unwrap_or_default(); if !chat_tool_call_is_read_only(&call) && !allow_mutation { bail!( "MCP tool `{name}` changes local ROCm state; rerun `rocm mcp-call {name}` with --allow-mutation only after explicit user approval" @@ -388,14 +388,60 @@ pub(crate) fn run_internal_mcp_call( } } -fn internal_mcp_arguments(value: serde_json::Value) -> serde_json::Map { - if let Some(arguments) = value +/// The validated tool call `run_internal_mcp_call` executes for `arguments`. +/// +/// `arguments` is used exactly as given: it is the object the dash approval +/// card was rendered from (`dash_seam::BinToolExecutor`), so validating and +/// running anything else — such as an object nested inside it — would run a +/// command the user never saw. `validate_chat_tool_call` refuses a nested +/// `arguments` key outright, so no such object reaches either side. +pub(crate) fn internal_mcp_tool_call( + name: &str, + arguments: serde_json::Value, +) -> Result { + let call = providers::ChatToolCall { + id: None, + name: name.to_owned(), + arguments, + }; + validate_chat_tool_call(&call)?; + Ok(call) +} + +/// The tool arguments for `rocm mcp-call --arguments-json `. +/// +/// Besides a bare arguments object, the hidden command accepts a pasted MCP +/// `tools/call` params envelope — `{"name": ..., "arguments": {...}}` — and +/// unwraps it here, at the CLI boundary, where the JSON is the operator's own. +/// An envelope holds only `name`, `arguments` and the protocol's optional +/// `_meta`, which carries request metadata rather than tool input and is +/// dropped. Only such a pure envelope is unwrapped: an object that mixes tool +/// fields with an `arguments` key is passed through whole, so validation +/// refuses it rather than silently picking one of the two payloads. +pub(crate) fn mcp_call_cli_arguments( + name: &str, + value: serde_json::Value, +) -> Result { + let Some(object) = value.as_object() else { + return Ok(value); + }; + let is_envelope = object .get("arguments") - .and_then(serde_json::Value::as_object) + .is_some_and(serde_json::Value::is_object) + && object + .keys() + .all(|key| matches!(key.as_str(), "name" | "arguments" | "_meta")); + if !is_envelope { + return Ok(value); + } + if let Some(envelope_name) = object.get("name") + && envelope_name.as_str() != Some(name) { - return arguments.clone(); + bail!( + "arguments JSON names tool {envelope_name}, but `rocm mcp-call` was asked to run `{name}`" + ); } - value.as_object().cloned().unwrap_or_default() + Ok(object["arguments"].clone()) } fn internal_mcp_tool_success(text: String, structured: serde_json::Value) -> serde_json::Value { @@ -755,8 +801,11 @@ pub(super) fn run_chat_read_only_tool( )) } "install_sdk_dry_run" => { - let arguments = internal_mcp_arguments(call.arguments.clone()); - let args = internal_mcp_install_sdk_args(&arguments, true)?; + let arguments = call + .arguments + .as_object() + .context("install_sdk_dry_run arguments must be a JSON object")?; + let args = internal_mcp_install_sdk_args(arguments, true)?; let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; Ok(internal_mcp_tool_result_from_command( "Ran `rocm install sdk --dry-run`.", @@ -935,6 +984,120 @@ mod tests { assert_eq!(content["lines_requested"], 50, "{content}"); } + #[test] + fn mcp_call_cli_unwraps_only_a_pure_params_envelope() { + // A pasted `tools/call` params object is unwrapped to its arguments. + assert_eq!( + mcp_call_cli_arguments( + "stop_server", + serde_json::json!({ + "name": "stop_server", + "arguments": { "service_id": "svc-1" }, + }), + ) + .expect("envelope unwraps"), + serde_json::json!({ "service_id": "svc-1" }) + ); + assert_eq!( + mcp_call_cli_arguments( + "stop_server", + serde_json::json!({ "arguments": { "service_id": "svc-1" } }), + ) + .expect("envelope without a name unwraps"), + serde_json::json!({ "service_id": "svc-1" }) + ); + // The protocol's optional `_meta` does not stop an envelope being one; + // it is request metadata and is dropped with the wrapper. + assert_eq!( + mcp_call_cli_arguments( + "stop_server", + serde_json::json!({ + "name": "stop_server", + "arguments": { "service_id": "svc-1" }, + "_meta": { "progressToken": 1 }, + }), + ) + .expect("envelope with _meta unwraps"), + serde_json::json!({ "service_id": "svc-1" }) + ); + // A bare arguments object passes through untouched. + let bare = serde_json::json!({ "service_id": "svc-1" }); + assert_eq!( + mcp_call_cli_arguments("stop_server", bare.clone()).expect("bare passes"), + bare + ); + // An envelope naming a different tool is refused rather than run as + // the tool on the command line. + assert!( + mcp_call_cli_arguments( + "stop_server", + serde_json::json!({ + "name": "launch_server", + "arguments": { "service_id": "svc-1" }, + }), + ) + .is_err() + ); + // A `name` that is not a string cannot name the tool on the command + // line, so the envelope is refused rather than unwrapped. + assert!( + mcp_call_cli_arguments( + "stop_server", + serde_json::json!({ "name": 5, "arguments": { "service_id": "svc-1" } }), + ) + .is_err() + ); + // An `arguments` that is not an object is not an envelope: the value + // passes through whole, and validation refuses its nested field. + let scalar = serde_json::json!({ "arguments": 5 }); + assert_eq!( + mcp_call_cli_arguments("stop_server", scalar.clone()).expect("passes through"), + scalar + ); + } + + #[test] + fn mcp_call_refuses_arguments_that_are_not_an_object() { + // The replay runs the value it is given, so a value that is not an + // object is refused rather than read as an empty argument object. + let (root, paths) = test_paths("mcp-call-non-object-arguments"); + for value in [ + serde_json::Value::Null, + serde_json::json!([]), + serde_json::json!("s"), + ] { + let passed = mcp_call_cli_arguments("examine", value.clone()).expect("passes through"); + assert_eq!(passed, value); + let err = run_internal_mcp_call(&paths, "examine", passed, false) + .expect_err("non-object arguments must be refused"); + assert!( + err.to_string().contains("must be a JSON object"), + "unexpected error for {value}: {err}" + ); + } + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn mcp_call_refuses_tool_fields_mixed_with_a_nested_arguments_object() { + // Mixed objects are not an envelope: they reach validation whole, and + // validation refuses them instead of choosing one of the two payloads. + let mixed = serde_json::json!({ + "service_id": "shown-service", + "arguments": { "service_id": "other-service" }, + }); + let passed = mcp_call_cli_arguments("stop_server", mixed.clone()).expect("passes through"); + assert_eq!(passed, mixed); + let (root, paths) = test_paths("mcp-call-nested-arguments"); + let err = run_internal_mcp_call(&paths, "stop_server", passed, true) + .expect_err("a nested arguments object must be refused before running"); + assert!( + err.to_string().contains("nested `arguments`"), + "unexpected error: {err}" + ); + let _ = fs::remove_dir_all(&root); + } + #[test] fn mcp_tool_result_text_joins_text_content_items() { let value = serde_json::json!({ diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 6104963818..55f8eb1579 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -236,7 +236,18 @@ pub(crate) enum Command { /// problem and changes nothing (`info-20-gpu-no-rocm-support` is the entry /// marked that way today: a GPU ROCm has no runtime build for is a fact /// about the hardware, not something to put right). - /// Use `--dry-run` to see any fix's plan without changing anything. + /// Use `--dry-run` to see a fix's plan without changing anything. + /// + /// A fix whose command fails exits 4 and says why on stderr. A command that + /// does not finish within its time limit is stopped and also exits 4; the + /// change may then be partial, so the message names how to check it and the + /// `rocm fix` command to re-run. On Windows, `fix-2-unset-override`, + /// `fix-6-path` and `fix-9-igpu-dgpu --device-index N` read the current + /// value of the variable they change before planning; when that read + /// fails they stop before changing anything, with exit 4. Under + /// `--dry-run` they stop the same way and show no plan; the catalog entry + /// printed first is the fix's generic description, not a preview of the + /// change on this machine. Fix { /// Fix id, e.g. fix-4-render-group. Omit to list available fixes. fix_id: Option, @@ -247,7 +258,11 @@ pub(crate) enum Command { /// Fixes that only print steps never ask. #[arg(long)] yes: bool, - /// Show the plan without changing anything. + /// Show the plan without changing anything. On Windows, a dry run of a + /// fix that reads the current value of the variable it would change + /// stops with exit 4 and shows no plan when that read fails; the + /// catalog entry printed first is the fix's generic description, not a + /// preview of the change on this machine. #[arg(long = "dry-run")] dry_run: bool, /// For fix-9-igpu-dgpu: the discrete GPU index to pin. @@ -560,6 +575,9 @@ rocm serve qwen --verbose --device gpu_required")] #[arg(long, conflicts_with = "managed")] verbose: bool, /// Skip the post-startup inference smoke test (time-to-first-token, tokens/sec). + /// + /// When the smoke test runs and cannot complete, the summary says so + /// and does not report the service as ready; ROCm CLI does not stop the server. #[arg(long)] no_smoke_test: bool, /// Allow binding to a non-local address. @@ -981,7 +999,17 @@ rocm engines install vllm --reinstall")] #[derive(Subcommand, Debug)] pub(crate) enum RuntimesCommand { /// Show ROCm runtimes known to ROCm CLI. - List, + /// + /// Each runtime is listed with its version (and build date), GPU family, + /// mode and toolchain, then its channel, package source, source layout + /// generation and exact GPU target as its manifest recorded them. A fact + /// recorded nowhere, or recorded blank, is shown as `unrecorded`; nothing + /// is guessed from other facts. + List { + /// Print the runtime list as JSON instead of text. + #[arg(long)] + json: bool, + }, /// Use the selected ROCm runtime by default. Activate { /// Runtime key or friendly runtime selector. @@ -1232,9 +1260,9 @@ pub(crate) enum ServicesCommand { /// /// Only works on a record that is not running: stop the server first (on /// Windows a process `rocm` cannot query counts as running). The record's - /// details file, log, engine state file, and endpoint key file are all - /// deleted, so its log can no longer be read and it can no longer be - /// restarted. + /// details file, log, engine state file, and endpoint key file (with any + /// copy of the key an interrupted save left beside it) are all deleted, so + /// its log can no longer be read and it can no longer be restarted. Remove { /// Service id from `rocm services list --all`. service_id: String, @@ -1245,9 +1273,9 @@ pub(crate) enum ServicesCommand { /// /// Only when a stop was requested, no recorded process is confirmed /// running, and at least one could not be queried (on Windows, a live - /// process the account may not open). Deletes the record, log, engine - /// state and endpoint key, and says which process; if it is still - /// running it keeps running. Refused for a process confirmed running. + /// process the account may not open). Deletes the same files as a + /// plain `remove`, and says which process; if it is still running it + /// keeps running. Refused for a process confirmed running. #[arg(long)] force: bool, }, @@ -2124,6 +2152,24 @@ mod tests { } } + /// `--json` belongs to `list` alone: bare `rocm runtimes` stays the text + /// listing, so a script has to name the subcommand to get JSON. + #[test] + fn runtimes_json_is_a_list_flag() { + for (args, json) in [(&["list", "--json"][..], true), (&["list"][..], false)] { + match parse_runtimes(args).expect("runtimes list parses").command { + Some(Command::Runtimes { + command: Some(RuntimesCommand::List { json: parsed }), + }) => assert_eq!(parsed, json, "{args:?}"), + other => panic!("expected runtimes list, got {other:?}"), + } + } + assert!( + parse_runtimes(&["--json"]).is_err(), + "`rocm runtimes --json` must be rejected, not read as `list --json`" + ); + } + fn parse_serve(args: &[&str]) -> Result { let mut argv = vec!["rocm", "serve", "qwen"]; argv.extend_from_slice(args); diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 70956108bb..3743607457 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -408,9 +408,28 @@ pub(super) fn runtime_environment_assignments( if let Some(path) = prepend_env_paths(&path_entries, std::env::var_os("PATH"))? { values.push(("PATH", path)); } + // The amd-smi link a managed runtime's torch needs to count devices (see + // `rocm_core::ensure_amd_smi_compat`), after the recorded directories. Linux + // only, like the `LD_LIBRARY_PATH` it goes on: other platforms never write it. + // A failure to write the link is deliberately not an error here. The link only + // repairs torch's device count beside a host ROCm; without it ComfyUI starts + // as it did before this change, and a launch should not be refused for a + // best-effort shim. The failure is not lost: `rocm examine` reports a runtime + // that needs the link and lacks it (see `amd_smi_link_missing`). + let mut library_entries = env.library_entries.clone(); + if runtime_is_linux() + && let Some(shim) = env.venv_root.as_ref().and_then(|venv_root| { + rocm_core::ensure_amd_smi_compat( + &venv_root.join(rocm_core::RUNTIME_COMPAT_DIR_NAME), + &env.library_entries, + ) + }) + { + library_entries.push(shim); + } if runtime_is_linux() && let Some(ld_library_path) = - prepend_env_paths(&env.library_entries, std::env::var_os("LD_LIBRARY_PATH"))? + prepend_env_paths(&library_entries, std::env::var_os("LD_LIBRARY_PATH"))? { values.push(("LD_LIBRARY_PATH", ld_library_path)); } @@ -1958,6 +1977,58 @@ mod tests { Ok(()) } + /// A second amd-smi library in the ComfyUI process makes torch count zero + /// devices; the launch environment must carry the link to the SDK's own. + #[cfg(target_os = "linux")] + #[test] + fn runtime_environment_carries_the_amd_smi_link_dir() -> Result<()> { + let paths = test_paths("comfyui-amd-smi-link"); + let venv_root = paths.data_dir.join("venv"); + let sdk_lib = paths.data_dir.join("sdk-lib"); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + // A `libmpi_cxx` stub a vLLM launch left in the same directory, which + // reaches this launch's loader path along with the link. + let compat = venv_root.join(rocm_core::RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&compat)?; + let stub = compat.join(rocm_core::openmpi::MPI_CXX_STUB_NAME); + fs::write(&stub, b"stub")?; + let env = ComfyUiRuntimeEnvironment { + venv_root: Some(venv_root), + library_entries: vec![sdk_lib.clone()], + ..Default::default() + }; + + let assignments = runtime_environment_assignments(&env)?; + + let ld_library_path = assignments + .iter() + .find(|(key, _)| *key == "LD_LIBRARY_PATH") + .map(|(_, value)| value.clone()) + .context("LD_LIBRARY_PATH was not assigned")?; + let entries: Vec = std::env::split_paths(&ld_library_path).collect(); + // After the SDK's own directory, so a real bare name found there wins. + let position = |dir: &Path| entries.iter().position(|entry| entry == dir); + assert!( + position(&sdk_lib).unwrap() < position(&compat).context("link dir missing")?, + "the link directory must come after the SDK's: {entries:?}" + ); + assert_eq!( + fs::read_link(compat.join("libamd_smi.so"))?, + sdk_lib.join("libamd_smi.so.26") + ); + // The stub shadows a real host library on that path, so it is gone where + // the host has one. Without one it may be the library torch loads, and stays. + // Only the half that matches this host runs here; both halves, with the host + // answer injected, are tested in rocm-core + // (`the_shared_helpers_clear_a_leftover_mpi_cxx_stub_only_beside_a_real_library`). + assert_eq!( + stub.exists(), + !rocm_core::openmpi::real_libmpi_cxx_present() + ); + Ok(()) + } + fn minimal_runtime_manifest(python: &Path) -> therock::InstalledRuntimeManifest { therock::InstalledRuntimeManifest { runtime_key: "therock-release-gfx120x-all".to_owned(), diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index c09f55f3c9..72a3624669 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -212,9 +212,11 @@ fn runtime_summaries(paths: &AppPaths, config: &RocmCliConfig) -> Vec = manifests .iter() .filter(|m| Some(m.runtime_id.as_str()) == default_id) diff --git a/apps/rocm/src/dash_seam.rs b/apps/rocm/src/dash_seam.rs index 2ac4ca4d91..cd5eaef038 100644 --- a/apps/rocm/src/dash_seam.rs +++ b/apps/rocm/src/dash_seam.rs @@ -319,4 +319,190 @@ mod tests { "approved-path execution must still reject an unsafe call, got {outcome:?}" ); } + + /// One approval-gated tool call per mutating tool, each carrying a second, + /// *different* payload under a nested `arguments` key. The outer object is + /// what the model's call looks like on its face; the inner one differs in + /// the fields the approval card shows, so any path that reads one object + /// for the card and the other for the command would be visible. The one + /// shared value is `install_sdk`'s `format`: `wheel` is the only format + /// valid on every platform, and the channel and prefix already differ. + fn nested_arguments_calls() -> Vec<(&'static str, serde_json::Value)> { + vec![ + ( + "launch_server", + serde_json::json!({ + "model": "shown/model", + "arguments": { "model": "other/model", "port": 8123 }, + }), + ), + ( + "install_sdk", + serde_json::json!({ + "channel": "release", + "format": "wheel", + "prefix": "/tmp/rocm-seam-shown-prefix", + "arguments": { + "channel": "nightly", + "format": "wheel", + "prefix": "/tmp/rocm-seam-other-prefix", + }, + }), + ), + ( + "install_engine", + serde_json::json!({ + "engine": "lemonade", + "arguments": { "engine": "vllm", "reinstall": true }, + }), + ), + ( + "stop_server", + serde_json::json!({ + "service_id": "shown-service", + "arguments": { "service_id": "other-service" }, + }), + ), + ( + "watcher_enable", + serde_json::json!({ + "watcher": "therock-update", + "mode": "observe", + "arguments": { "watcher": "server-recover", "mode": "contained" }, + }), + ), + ( + "watcher_disable", + serde_json::json!({ + "watcher": "therock-update", + "arguments": { "watcher": "server-recover" }, + }), + ), + ] + } + + /// The command line the approved replay of `arguments` would run, read + /// through the two calls `run_internal_mcp_call`'s mutating arm makes + /// (`internal_mcp_tool_call`, then `rocm_chat_tool_requested_args`) + /// without spawning it. + fn approved_command_line(name: &str, arguments: &serde_json::Value) -> Option { + let call = chat::internal_mcp_tool_call(name, arguments.clone()).ok()?; + let args = chat::rocm_chat_tool_requested_args(&call)?; + Some(crate::rocm_argv::format_structured_tool_call("rocm", &args)) + } + + /// The approval card for a mutating tool and the command its approval runs + /// must describe the same action. For every approval-gated tool, a call + /// carrying a nested `arguments` payload either never reaches a card, or + /// the command run on approval is exactly the one the card shows. + /// + /// This is the statement of the guarantee over the shared command builder + /// (`approved_command_line`): with both layers of the fix undone there (no + /// refusal, and `internal_mcp_tool_call` unwrapping the nested object) it + /// reports a disagreement for every tool. Either layer alone keeps it + /// green, by design, so each is pinned by its own test. Removing the + /// refusal fails `seam_refuses_nested_arguments_before_approval_and_on_replay` + /// and `validate_chat_tool_call_refuses_a_nested_arguments_field_for_every_tool`. + /// An unwrap added ahead of validation, in `internal_mcp_tool_call` or in + /// `run_internal_mcp_call` before it, fails + /// `seam_refuses_nested_arguments_before_approval_and_on_replay`, whose + /// approved replay then runs instead of being refused, and + /// `mcp_call_refuses_tool_fields_mixed_with_a_nested_arguments_object` in + /// `chat/mcp_bridge.rs`. + #[test] + fn seam_approval_card_matches_the_command_approval_runs() { + let exec = BinToolExecutor::new(temp_paths()); + let mut disagreements = Vec::new(); + for (name, payload) in nested_arguments_calls() { + match exec.execute(name, &payload) { + RocmToolOutcome::Error(_) => {} + RocmToolOutcome::ApprovalRequired(intent) => { + let ran = approved_command_line(&intent.name, &intent.arguments); + if !ran.as_ref().is_some_and(|ran| intent.body.contains(ran)) { + disagreements.push(format!( + "`{name}`: the approval card shows {:?} but approving it runs {ran:?}", + intent.body + )); + } + } + RocmToolOutcome::Result(v) => { + panic!("`{name}` is approval-gated but ran without approval: {v}") + } + } + } + assert!(disagreements.is_empty(), "{}", disagreements.join("\n")); + } + + /// The same calls with the nested payload removed are ordinary, valid + /// requests: each must still reach an approval card, and that card must + /// show exactly the command its approval runs. Without this the agreement + /// test above could pass by refusing every call. + #[test] + fn seam_approval_card_matches_the_command_for_plain_calls() { + let exec = BinToolExecutor::new(temp_paths()); + for (name, mut payload) in nested_arguments_calls() { + payload + .as_object_mut() + .expect("fixture is an object") + .remove("arguments"); + let RocmToolOutcome::ApprovalRequired(intent) = exec.execute(name, &payload) else { + panic!("plain `{name}` call must reach an approval card"); + }; + assert_eq!( + intent.arguments, payload, + "`{name}`: the intent replays the object the card was built from" + ); + let ran = approved_command_line(&intent.name, &intent.arguments) + .unwrap_or_else(|| panic!("plain `{name}` call must be runnable once approved")); + assert!( + intent.body.contains(&ran), + "`{name}`: the approval card shows {:?} but approving it runs {ran}", + intent.body + ); + } + } + + /// A nested `arguments` payload is refused before any card is shown, and + /// the approved replay refuses it too, without spawning anything — so an + /// intent assembled some other way still cannot run the inner payload. + #[test] + fn seam_refuses_nested_arguments_before_approval_and_on_replay() { + let exec = BinToolExecutor::new(temp_paths()); + for (name, payload) in nested_arguments_calls() { + match exec.execute(name, &payload) { + RocmToolOutcome::Error(e) => assert!( + e.contains("nested `arguments`"), + "`{name}`: refusal should name the nested field, got: {e}" + ), + other => panic!("`{name}` with a nested payload must be refused, got {other:?}"), + } + match exec.execute_approved(name, &payload) { + RocmToolOutcome::Error(e) => assert!( + e.contains("nested `arguments`"), + "`{name}`: replay refusal should name the nested field, got: {e}" + ), + other => panic!( + "`{name}` replay with a nested payload must be refused before running, got {other:?}" + ), + } + } + } + + /// Read-only tools go through the same gate: the result returned to the + /// model must describe the call the model made, not an object inside it. + #[test] + fn seam_refuses_nested_arguments_on_read_only_tools() { + let exec = BinToolExecutor::new(temp_paths()); + let outcome = exec.execute( + "rocm_command", + &serde_json::json!({ + "args": ["engines"], + "arguments": { "args": ["runtimes", "activate", "some-runtime"] }, + }), + ); + assert!( + matches!(&outcome, RocmToolOutcome::Error(e) if e.contains("nested `arguments`")), + "read-only call with a nested payload must be refused, got {outcome:?}" + ); + } } diff --git a/apps/rocm/src/driver_plan.rs b/apps/rocm/src/driver_plan.rs index 44cd8810e5..1ef2804685 100644 --- a/apps/rocm/src/driver_plan.rs +++ b/apps/rocm/src/driver_plan.rs @@ -1279,6 +1279,7 @@ pub(crate) mod tests { }), managed_runtime_count: 0, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("/tmp/config"), data_dir: PathBuf::from("/tmp/data"), diff --git a/apps/rocm/src/endpoint_keys.rs b/apps/rocm/src/endpoint_keys.rs index e85fc3d216..e7c224db7c 100644 --- a/apps/rocm/src/endpoint_keys.rs +++ b/apps/rocm/src/endpoint_keys.rs @@ -6,48 +6,28 @@ //! //! When `rocm serve` binds a public (non-loopback) interface it protects the //! endpoint with an API key (see `resolve_endpoint_auth` in `service_launch/endpoint_auth.rs`). -//! The key is stored in a 0600 file under the services directory, keyed by -//! service id. +//! The key is stored in an owner-only file under the services directory, keyed +//! by service id: mode 0600 on Unix, and on Windows a protected DACL granting +//! only the user who stored it, so the data directory's own ACL is not +//! inherited. Every store replaces the file with a freshly created owner-only +//! one, so a key file whose permissions were loosened is tightened again rather +//! than rewritten in place (see `write_owner_only_file_atomically` in +//! `rocm_core::atomic_write`). //! //! A file — not the OS keychain — is deliberate: public binding is overwhelmingly //! a *headless server* action (that is what `0.0.0.0` is for), and headless Linux //! hosts routinely have no Secret Service / D-Bus session, so a keychain-backed //! store would make `rocm serve --host 0.0.0.0` fail on its primary platform. An -//! owner-only (0600) file under `~/.rocm` satisfies "not persisted insecurely" +//! owner-only file under `~/.rocm` satisfies "not persisted insecurely" //! while working the same everywhere, and is the same shape (`--api-key-file`) //! engines like llama-server already use. use anyhow::{Context, Result}; -use std::fs; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use rocm_core::AppPaths; -/// Write `contents` to `path` with owner-only (0600) permissions on Unix so a -/// secret is not world-readable. On non-Unix, default permissions apply. -fn write_private_file_0600(path: &Path, contents: &[u8]) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(path)?; - std::io::Write::write_all(&mut file, contents)?; - } - #[cfg(not(unix))] - { - fs::write(path, contents)?; - } - Ok(()) -} - -/// Deterministic path of the 0600 endpoint key file for `service_id`. +/// Deterministic path of the owner-only endpoint key file for `service_id`. /// /// Delegates to the shared helper in `rocm-engine-protocol` so `rocmd` (which /// needs the same path to re-thread `ROCM_SERVE_API_KEY_FILE` onto engine @@ -57,11 +37,12 @@ pub(crate) fn endpoint_key_file_path(paths: &AppPaths, service_id: &str) -> Path rocm_engine_protocol::endpoint_key_file_path(paths, service_id) } -/// Persist `key` for `service_id`, creating the file 0600. Overwrites any -/// existing value. +/// Persist `key` for `service_id` in a file only the current user can read +/// (0600 on Unix, a protected owner-only DACL on Windows). Replaces any +/// existing file, together with whatever permissions it had. pub(crate) fn store_endpoint_api_key(paths: &AppPaths, service_id: &str, key: &str) -> Result<()> { let path = endpoint_key_file_path(paths, service_id); - write_private_file_0600(&path, key.as_bytes()) + rocm_core::atomic_write::write_owner_only_file_atomically(&path, key.as_bytes()) .with_context(|| format!("failed to write endpoint API key file {}", path.display())) } @@ -80,11 +61,13 @@ pub(crate) fn endpoint_key_file_if_present(paths: &AppPaths, service_id: &str) - rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) } -/// Remove the stored key for `service_id`. Idempotent and best-effort: a missing -/// file is not an error, so this is safe to call unconditionally when any service -/// stops. +/// Remove the stored key for `service_id`, with any staged copy a store killed +/// before publishing left beside it. Idempotent and best-effort: a missing file +/// is not an error, and a file that cannot be removed is left without being +/// reported, so this is safe to call unconditionally when any service stops. +/// It does not exclude a concurrent store for the same service. pub(crate) fn clear_endpoint_api_key(paths: &AppPaths, service_id: &str) { - let _ = std::fs::remove_file(endpoint_key_file_path(paths, service_id)); + rocm_engine_protocol::remove_endpoint_key_files(paths, service_id); } #[cfg(test)] @@ -108,6 +91,29 @@ mod tests { ) } + /// A store killed between staging the key and publishing it leaves a + /// second copy beside the key file; clearing the key takes that copy too. + #[test] + fn clearing_a_key_removes_a_staged_copy_a_killed_store_left() { + let (root, paths) = temp_paths("staged-leftover"); + store_endpoint_api_key(&paths, "svc-staged", "secret-key").unwrap(); + let staged = rocm_core::atomic_write::stage_owner_only_file_unpublished( + &endpoint_key_file_path(&paths, "svc-staged"), + b"secret-key", + ) + .unwrap(); + + clear_endpoint_api_key(&paths, "svc-staged"); + let staged_exists = staged.exists(); + let key_exists = endpoint_key_file_path(&paths, "svc-staged").exists(); + let _ = std::fs::remove_dir_all(&root); + assert!(!key_exists, "the key file must be removed"); + assert!( + !staged_exists, + "the staged copy of the key must be removed too" + ); + } + #[test] fn temp_paths_gives_each_call_its_own_root_even_for_one_label() { crate::test_support::assert_each_call_gets_its_own_root(|| temp_paths("same-label").0); @@ -152,4 +158,43 @@ mod tests { assert_eq!(mode & 0o777, 0o600, "endpoint key file must be 0600"); let _ = std::fs::remove_dir_all(&root); } + + /// Re-storing a key over a file someone loosened leaves it 0600 again, + /// with the new key in it: the old mode is not kept (a restart re-stores the + /// key over the file of the previous run). + #[cfg(unix)] + #[test] + fn restoring_a_key_over_a_loosened_file_makes_it_0600_again() { + use std::os::unix::fs::PermissionsExt as _; + let (root, paths) = temp_paths("loosened"); + let path = endpoint_key_file_path(&paths, "svc-loose"); + std::fs::write(&path, "old-key").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + store_endpoint_api_key(&paths, "svc-loose", "new-key").unwrap(); + + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + let stored = endpoint_api_key(&paths, "svc-loose"); + let _ = std::fs::remove_dir_all(&root); + assert_eq!(mode, 0o600, "endpoint key file was left at {mode:o}"); + assert_eq!(stored, Some("new-key".to_owned())); + } + + /// On Windows the stored key has a protected DACL granting only the user + /// who stored it, whatever the data directory hands down. + #[cfg(windows)] + #[test] + fn stored_key_file_has_an_owner_only_dacl() { + let (root, paths) = temp_paths("dacl"); + store_endpoint_api_key(&paths, "svc-dacl", "secret-key").unwrap(); + let mismatch = rocm_core::owner_only_file::owner_only_dacl_mismatch( + &endpoint_key_file_path(&paths, "svc-dacl"), + ); + let _ = std::fs::remove_dir_all(&root); + assert_eq!( + mismatch.unwrap(), + None, + "the key file's DACL is not owner-only" + ); + } } diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 3c061bc26f..0735f9b68f 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -28,8 +28,9 @@ //! (`engine_request`, `engine_request_with_env_root`, //! `engine_stdio_child_request`), the `ScopedEnvVar`/ //! `with_scoped_builtin_engine_env` env-scoping helpers and the -//! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, -//! `run_builtin_engine_stdio`, and the built-in HTTP-serving path +//! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_handler`, +//! `builtin_engine_request`, `run_builtin_engine_stdio` with +//! `answer_builtin_engine_stdio`, and the built-in HTTP-serving path //! (`builtin_engine_available`, `run_builtin_engine_serve_http`, //! `builtin_engine_serve_http_args`). //! @@ -238,35 +239,51 @@ fn with_scoped_builtin_engine_env( result } +/// How a built-in engine answers a decoded request. +type BuiltinEngineHandler = fn(EngineRequestEnvelope) -> EngineResponseEnvelope; + +/// The handler of the engine built into this binary under `engine`, or `None`. +/// The in-process request path, `rocm __engine-stdio` and +/// [`builtin_engine_available`] all ask it; `run_builtin_engine_serve_http` +/// keeps its own match over the same engines. +fn builtin_engine_handler(engine: &str) -> Option { + match engine { + "lemonade" => Some(rocm_engine_lemonade::builtin_handle_envelope), + "vllm" => Some(rocm_engine_vllm::builtin_handle_envelope), + "aim" => Some(rocm_engine_aim::builtin_handle_envelope), + _ => None, + } +} + fn builtin_engine_request( engine: &str, envelope: &EngineRequestEnvelope, ) -> Option { - match engine { - "lemonade" => Some(rocm_engine_lemonade::builtin_handle_envelope( - envelope.clone(), - )), - "vllm" => Some(rocm_engine_vllm::builtin_handle_envelope(envelope.clone())), - "aim" => Some(rocm_engine_aim::builtin_handle_envelope(envelope.clone())), - _ => None, - } + builtin_engine_handler(engine).map(|handle| handle(envelope.clone())) } pub(crate) fn run_builtin_engine_stdio(engine: &str) -> Result<()> { + let handle = builtin_engine_handler(engine) + .with_context(|| format!("engine `{engine}` is not built into this rocm binary"))?; let mut input = String::new(); io::stdin() .read_to_string(&mut input) .context("failed to read engine stdio request")?; - let envelope: EngineRequestEnvelope = - serde_json::from_str(&input).context("failed to parse engine stdio request")?; - let response = builtin_engine_request(engine, &envelope) - .with_context(|| format!("engine `{engine}` is not built into this rocm binary"))?; + let response = answer_builtin_engine_stdio(handle, &input); print!("{}", serde_json::to_string(&response)?); Ok(()) } +/// The answer a built-in engine's `handle` gives to the request `text` read on +/// stdin: the version is judged before the method or payload is decoded (see +/// [`rocm_engine_protocol::answer_engine_request`]), as a separately built +/// engine's `stdio` command does. +fn answer_builtin_engine_stdio(handle: BuiltinEngineHandler, text: &str) -> EngineResponseEnvelope { + rocm_engine_protocol::answer_engine_request(text, handle) +} + pub(crate) fn builtin_engine_available(engine: &str) -> bool { - matches!(engine, "lemonade" | "vllm" | "aim") + builtin_engine_handler(engine).is_some() } #[allow(clippy::too_many_arguments)] @@ -412,6 +429,73 @@ mod tests { } } + /// Every engine built into this binary holds the engine protocol contract on + /// both of its paths: the decoded request `rocm` hands it in-process, and + /// `answer_builtin_engine_stdio`, the function `rocm __engine-stdio ` + /// (which `rocmd` runs) answers request text with. The verb's own stdin + /// reading is not run here; e2e engine-ipc-08 to -12 cover it, for `vllm` + /// only. A refusal reaches the caller through the same + /// `decode_engine_response` as a child's answer, naming the engine's code. + #[test] + fn every_built_in_engine_holds_the_protocol_contract() { + assert!(!builtin_engine_available("not-an-engine")); + assert!( + builtin_engine_request( + "not-an-engine", + &EngineRequestEnvelope::new(EngineMethod::Capabilities, serde_json::json!({})) + ) + .is_none() + ); + for engine in crate::cli::SUPPORTED_ENGINES { + assert!(builtin_engine_available(engine), "{engine}"); + let handle = builtin_engine_handler(engine).expect("built in"); + rocm_engine_protocol::engine_contract::assert_engine_holds_protocol_contract( + engine, + |text| answer_builtin_engine_stdio(handle, text), + |envelope| builtin_engine_request(engine, &envelope).expect("built in"), + ); + let mut request = EngineRequestEnvelope::new(EngineMethod::Stop, serde_json::json!({})); + request.protocol_version = Some("9.0.0".to_owned()); + let refusal = builtin_engine_request(engine, &request).expect("built in"); + assert_eq!( + decode_engine_response::(refusal) + .unwrap_err() + .to_string(), + format!( + "incompatible_protocol: this engine speaks engine protocol \ + {}; the request was made for protocol 9.0.0, which \ + it cannot serve", + rocm_engine_protocol::ENGINE_PROTOCOL_VERSION + ), + "{engine}" + ); + } + } + + /// Each built-in engine id reaches that engine's own handler: its + /// capabilities answer is the one the engine crate itself gives, and the + /// three engines' answers differ, so a swapped mapping shows. + #[test] + fn each_built_in_engine_id_reaches_its_own_engine() { + let capabilities = + || EngineRequestEnvelope::new(EngineMethod::Capabilities, serde_json::json!({})); + let own: [(&str, BuiltinEngineHandler); 3] = [ + ("lemonade", rocm_engine_lemonade::builtin_handle_envelope), + ("vllm", rocm_engine_vllm::builtin_handle_envelope), + ("aim", rocm_engine_aim::builtin_handle_envelope), + ]; + let mut answers = Vec::new(); + for (engine, handle) in own { + let mapped = builtin_engine_handler(engine).expect("built in"); + let expected = handle(capabilities()).data; + assert_eq!(mapped(capabilities()).data, expected, "{engine}"); + answers.push(expected); + } + assert_ne!(answers[0], answers[1]); + assert_ne!(answers[1], answers[2]); + assert_ne!(answers[0], answers[2]); + } + #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = test_paths("engine-plugin"); @@ -491,7 +575,7 @@ mod tests { const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { - EngineRequestEnvelope { method, payload } + EngineRequestEnvelope::new(method, payload) } fn stop_request() -> EngineRequestEnvelope { @@ -538,7 +622,11 @@ mod tests { assert_eq!(engine.argv(), ["stdio"]); assert_eq!( engine.stdin(), - "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + format!( + "{{\"protocol_version\":\"{}\",\"method\":\"stop\",\ + \"payload\":{{\"service_id\":\"svc\"}}}}\n", + rocm_engine_protocol::ENGINE_PROTOCOL_VERSION + ) ); } diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index ec67be84b1..5045c384eb 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -326,6 +326,12 @@ pub(crate) fn append_examine_runtime_state( "managed" }; let _ = writeln!(output, " active_runtime_mode: {mode}"); + // Where the active runtime came from, from the same fields and in the + // same words as the `provenance:` line `rocm runtimes list` prints for + // it, so the two commands cannot describe one runtime differently. + for (name, value) in manifest.provenance().fields() { + let _ = writeln!(output, " active_runtime_{name}: {value}"); + } } if let Some((setup_root, pip_cache_dir)) = setup_runtime_paths(config) { let _ = writeln!(output, " setup_runtime_root: {}", setup_root.display()); @@ -614,6 +620,7 @@ mod tests { wsl: None, managed_runtime_count, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), diff --git a/apps/rocm/src/gpu_policy.rs b/apps/rocm/src/gpu_policy.rs index 108a642d35..064a4d41c0 100644 --- a/apps/rocm/src/gpu_policy.rs +++ b/apps/rocm/src/gpu_policy.rs @@ -149,7 +149,7 @@ pub(crate) fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { /// /// That span is relied on by a second caller: because the guard outlives both /// `store_endpoint_api_key` and the first `record.write()`, it also covers the -/// interval in which a launch's 0600 endpoint key sits on disk with no manifest +/// interval in which a launch's owner-only endpoint key sits on disk with no manifest /// beside it. [`crate::service_records::prune_managed_service_records`] acquires the same lock so its /// leftover sweep cannot read the directory in that state. Shortening the /// guard's life — releasing it before the record is persisted — would silently diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f3fc7f3faf..5d385cf235 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -757,6 +757,7 @@ fn dispatch(cli: Cli) -> Result<()> { }; let arguments = serde_json::from_str(arguments_json.trim_start_matches('\u{feff}')) .with_context(|| format!("failed to parse arguments JSON for `{name}`"))?; + let arguments = chat::mcp_call_cli_arguments(&name, arguments)?; let value = chat::run_internal_mcp_call(&paths, &name, arguments, allow_mutation)?; println!("{}", serde_json::to_string(&value)?); Ok(()) @@ -1893,7 +1894,7 @@ pub(crate) mod tests { Ok(record) } - /// Everything [`plant_service_record`] writes *except* the 0600 endpoint + /// Everything [`plant_service_record`] writes *except* the owner-only endpoint /// key: the manifest, the log, and the engine state file. /// /// Split out for the launch-lock test below, which writes the key itself — diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index ccd8826ef1..42c3a978c7 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -230,7 +230,7 @@ pub(crate) fn container_stop_unresolved(record: &ManagedServiceRecord) -> bool { /// operator never asked for it to go away and will want it back: dropping the /// key there would make `rocm services restart` and daemon recovery refuse it /// permanently — fail-closed guards have no way to re-mint a key. The cost of -/// this choice is that a crashed public service's 0600 key file outlives the +/// this choice is that a crashed public service's owner-only key file outlives the /// process until the service is stopped or successfully restarted. /// /// Called before the liveness gate in diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d35d738698..f3b9fd7e68 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1673,22 +1673,16 @@ mod tests { paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + write_chunked_sse_head(stream)?; let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; let done = "data: [DONE]\n\n"; write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", done.len(), done)?; stream.flush()?; Ok(()) - }, - ); + }); let mut ready = ManagedServiceRecord::new( &paths, @@ -1750,26 +1744,42 @@ mod tests { /// leaves behind. fn ready_service_that_dies_mid_chunked_stream( name: &str, + ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { + ready_service_answering_chat_with(name, answer_then_die_on_a_chunk_boundary) + } + + /// Sends the chunked SSE head and one content chunk, then returns, which + /// drops the stream: the client sees EOF on a chunk boundary. + fn answer_then_die_on_a_chunk_boundary(stream: &mut TcpStream) -> Result<()> { + write_chunked_sse_head(stream)?; + let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; + write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; + stream.flush()?; + Ok(()) + } + + fn write_chunked_sse_head(stream: &mut TcpStream) -> Result<()> { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + )?; + Ok(()) + } + + /// A ready local service whose engine answers the chat request with + /// `respond` (after answering any `/v1/models` listing itself). + fn ready_service_answering_chat_with( + name: &str, + mut respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { let (root, paths) = temp_app_paths(name); paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; - let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; - write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; - stream.flush()?; - // Returning drops the stream: the client sees EOF on a chunk boundary. - Ok(()) - }, - ); + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + respond(stream) + }); let mut ready = ManagedServiceRecord::new( &paths, "svc-ready", @@ -1888,30 +1898,31 @@ mod tests { } /// `rocm serve`'s post-launch smoke test is the CLI's streaming consumer. An - /// engine that dies mid-answer must leave the deployment summary showing no - /// measurements (`n/a`) instead of hanging the command or reporting - /// throughput for an answer that never finished. + /// engine that dies mid-answer must make the smoke test fail, and the + /// deployment summary built from that outcome must say so (heading, status + /// and reason, metrics `n/a`) instead of hanging the command, presenting the + /// service as ready, or reporting throughput for an answer that never + /// finished. #[test] - fn serve_smoke_test_reports_no_metrics_when_engine_dies_mid_chunked_stream() -> Result<()> { - let (root, paths, server) = - ready_service_that_dies_mid_chunked_stream("serve-smoke-chunked-truncated")?; - let (sender, receiver) = mpsc::channel(); - thread::spawn(move || { - let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); - }); - let (outcome, served) = client_then_server(&receiver, server); - fs::remove_dir_all(root).ok(); - - let metrics = outcome.map_err(|_| { - anyhow::anyhow!( - "the serve smoke test never returned after the engine closed the connection" - ) - })?; - // Empty metrics are also what a smoke test that never reached the - // engine returns, so this is what makes the assertion below mean + fn serve_smoke_test_fails_and_summary_says_so_when_engine_dies_mid_chunked_stream() -> Result<()> + { + // `smoke_outcome_against` also checks the engine side served its + // request: a failed outcome is also what a smoke test that never reached + // the engine returns, so that is what makes the assertion below mean // "the engine died mid-answer" rather than "nothing happened". - served.transpose()?; - assert_eq!(metrics, crate::serve_summary::SmokeMetrics::default()); + let smoke = smoke_outcome_against( + "serve-smoke-chunked-truncated", + answer_then_die_on_a_chunk_boundary, + )?; + // The asserts below do not interpolate the outcome or the rendered + // summary: `DeploymentSummary` carries an API key field, and echoing it + // trips CodeQL's `rust/cleartext-logging`. + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke.clone() else { + anyhow::bail!("a stream that died mid-answer was reported as measured"); + }; + // The reason is the provider's own truncation message, not just any text. + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); + let (metrics, smoke_failure) = smoke.into_parts(); let rendered = crate::serve_summary::render_summary(&crate::serve_summary::DeploymentSummary { engine: "vllm".to_owned(), @@ -1923,11 +1934,170 @@ mod tests { status: "ready".to_owned(), already_running: false, metrics, + smoke_failure, api_key: None, notes: Vec::new(), }); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("started, smoke test failed")); + assert!(!rendered.contains(" ready\n")); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains(&reason)); + assert!(rendered.contains("`rocm logs --service svc-ready`")); + Ok(()) + } + + /// Runs the smoke test against a ready service whose engine answers the chat + /// request with `respond`, and returns the outcome. + fn smoke_outcome_against( + name: &str, + respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, + ) -> Result { + let (root, paths, server) = ready_service_answering_chat_with(name, respond)?; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); + }); + let (outcome, served) = client_then_server(&receiver, server); + fs::remove_dir_all(root).ok(); + let smoke = outcome.map_err(|_| anyhow::anyhow!("the smoke test never returned"))?; + served.transpose()?; + Ok(smoke) + } + + /// A chat request that completes without error is not a failed smoke test + /// just because no `content` token arrived: a reasoning model can spend its + /// small token budget on `reasoning_content` first, and the readiness probe + /// (`openai_chat_completion_probe`) already treats such a server as serving. + /// The metrics stay empty (`n/a`) and no failure is recorded. + #[test] + fn serve_smoke_test_is_not_a_failure_when_a_complete_stream_carries_no_content() -> Result<()> { + for (name, body) in [ + ( + "serve-smoke-reasoning-only", + "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"thinking\"}}]}\n\ndata: [DONE]\n\n", + ), + ("serve-smoke-done-only", "data: [DONE]\n\n"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write_chunked_sse_head(stream)?; + write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", body.len(), body)?; + stream.flush()?; + Ok(()) + })?; + assert!( + smoke + == crate::serve_summary::SmokeOutcome::Measured( + crate::serve_summary::SmokeMetrics::default() + ), + "a complete stream with no content token was not a measured, empty outcome" + ); + let (metrics, smoke_failure) = smoke.into_parts(); + assert!(smoke_failure.is_none()); + assert!(metrics == crate::serve_summary::SmokeMetrics::default()); + } + Ok(()) + } + + /// The reason in the summary is built from the engine's own words (here its + /// HTTP status line), so it must reach the terminal as one clean line: no + /// escape sequence, no control or format character, but still the status. + #[test] + fn serve_smoke_test_failure_reason_carries_no_terminal_control_sequences() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-hostile-status", |stream| { + write!( + stream, + "HTTP/1.1 500 \u{1b}[31mBoom\u{7}\u{202e}x\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP 500 was not reported as a failure"); + }; + assert!(reason.contains("500")); + assert!(reason.contains("Boom")); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format), + "the failure reason still carries a control or format character" + ); + Ok(()) + } + + /// An engine that dies *inside* a chunk leaves an error with a cause chain + /// (the truncation message wrapping the failed read). The summary's reason + /// must keep the cause, not just the outermost message. + #[test] + fn serve_smoke_test_failure_reason_keeps_the_cause_chain() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-dies-inside-chunk", |stream| { + write_chunked_sse_head(stream)?; + // Promise 0x64 bytes, send three, close. + write!(stream, "64\r\nabc")?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an engine that died inside a chunk was not reported as a failure"); + }; + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); + assert!( + reason.contains("failed to read local provider SSE body"), + "the reason lost the cause behind the truncation message" + ); + Ok(()) + } + + /// Any HTTP error status from the engine is a failed smoke test, with the + /// status in the reason: the check is stricter than the readiness probe's + /// "anything below 500", so a 4xx fails here too. + #[test] + fn serve_smoke_test_fails_on_any_non_200_status() -> Result<()> { + for (name, status_line, code) in [ + ("serve-smoke-http-500", "500 Internal Server Error", "500"), + ("serve-smoke-http-401", "401 Unauthorized", "401"), + ("serve-smoke-http-404", "404 Not Found", "404"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write!( + stream, + "HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP error status was not reported as a failure"); + }; + assert!(reason.contains(code), "the reason lost the status code"); + } + Ok(()) + } + + /// A close-delimited stream (no chunking, no `Content-Length`) that ends + /// after some tokens is deliberately `Measured`: it has no framing to cut + /// short, so a clean close is indistinguishable from completion (see + /// `read_close_delimited_sse_body`). Pins `run_smoke_test`'s doc comment. + #[test] + fn serve_smoke_test_measures_a_close_delimited_stream_cut_after_a_token() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-close-delimited", |stream| { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n" + )?; + write!( + stream, + "data: {{\"choices\":[{{\"delta\":{{\"content\":\"hi\"}}}}]}}\n\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Measured(metrics) = smoke else { + anyhow::bail!("a close-delimited stream cut after a token was reported as failed"); + }; + assert!(metrics.ttft.is_some()); Ok(()) } diff --git a/apps/rocm/src/providers/tool_schema.rs b/apps/rocm/src/providers/tool_schema.rs index ee4fdff776..2ccb0c8ef9 100644 --- a/apps/rocm/src/providers/tool_schema.rs +++ b/apps/rocm/src/providers/tool_schema.rs @@ -189,3 +189,31 @@ pub(super) fn rocm_anthropic_tool_definitions() -> Vec { }) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + /// `validate_chat_tool_call` refuses any tool call carrying an `arguments` + /// field, which is only safe while no tool declares one. + #[test] + fn no_rocm_tool_declares_an_arguments_parameter() { + let tools = rocm_openai_tool_definitions(); + assert!(!tools.is_empty(), "the tool catalog is not empty"); + for tool in tools { + let function = &tool["function"]; + assert!( + function["parameters"]["properties"].is_object(), + "tool {} has a properties object", + function["name"] + ); + assert!( + function["parameters"]["properties"] + .get("arguments") + .is_none(), + "tool {} declares an `arguments` parameter", + function["name"] + ); + } + } +} diff --git a/apps/rocm/src/remote/provision.rs b/apps/rocm/src/remote/provision.rs index 5924262f52..be6a52b2b1 100644 --- a/apps/rocm/src/remote/provision.rs +++ b/apps/rocm/src/remote/provision.rs @@ -422,10 +422,11 @@ fn tempdir_for_download() -> Result { /// signature stops that, and today one is always required: `resolve_public_keys` /// falls back to the pinned release key with no channel argument, so /// `public_keys` is non-empty on every channel and the `|| [ -n "${public_keys}" ]` -/// arm at `install.sh:431` fires regardless of the channel-gated `require_sig` -/// above it. Clearing both pinned key slots is the only way to reach an unsigned -/// install. 0700 is still what keeps a second local user out of the gap in the -/// meantime — and what keeps this sound if a channel ever ships without a key. +/// arm of install.sh's signature check fires regardless of the channel-gated +/// `require_sig` before it. Clearing both pinned key slots is the only way to +/// reach an unsigned install. 0700 is still what keeps a second local user out +/// of the gap in the meantime — and what keeps this sound if a channel ever +/// ships without a key. /// /// (The signing key itself never lands here. It travels in the command prefix, /// and install.sh writes it into its own `mktemp -d` on the far side.) diff --git a/apps/rocm/src/remote/session.rs b/apps/rocm/src/remote/session.rs index bbbece29b7..184e9e0027 100644 --- a/apps/rocm/src/remote/session.rs +++ b/apps/rocm/src/remote/session.rs @@ -232,7 +232,8 @@ impl std::error::Error for NameAlreadyHeld {} /// key on its way out — leaving the holder serving a published tailnet endpoint /// that nobody, including its owner, can call. /// -/// `create_new` is what closes it: `O_EXCL` on unix, `CREATE_NEW` on Windows, +/// `create_new` is what closes it: `O_EXCL` on unix, `CREATE_NEW` on Windows +/// (both through [`rocm_core::owner_only_file::open_new_owner_only`]), /// and on both the existence check and the creation are one indivisible step. It /// is the same discipline [`super::publish`] applies to a port — establish /// ownership before writing, never after — with the difference that here the @@ -247,17 +248,10 @@ pub(crate) fn store_key(paths: &AppPaths, session_id: &str, key: &str) -> Result .with_context(|| format!("failed to create {}", directory.display()))?; let path = key_path(paths, session_id); - let mut options = fs::OpenOptions::new(); - options.write(true).create_new(true); - // 0600 at creation, not tightened afterwards: a credential must never exist, - // even briefly, at whatever the umask would have given it. - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(0o600); - } - - let mut file = match options.open(&path) { + // Owner-only at creation, not tightened afterwards: a credential must never + // exist, even briefly, at whatever the umask or the directory's ACL would + // have given it. 0600 on Unix, a protected owner-only DACL on Windows. + let mut file = match rocm_core::owner_only_file::open_new_owner_only(&path) { Ok(file) => file, Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { return Err(anyhow::Error::new(NameAlreadyHeld { path })); @@ -593,6 +587,14 @@ mod tests { let mode = fs::metadata(&path).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o600, "a credential must be owner-only"); } + // On Windows: a protected DACL granting only this user, not whatever + // the data directory would have handed down. + #[cfg(windows)] + assert_eq!( + rocm_core::owner_only_file::owner_only_dacl_mismatch(&path).expect("read the DACL"), + None, + "a credential must be owner-only" + ); clear_key(&paths, "remote-gpu-box-11434"); assert!(!path.exists()); diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 18e594f44b..d142bfc0ad 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -29,9 +29,9 @@ use rocm_core::{ }; use crate::runtime_registry::{ - ManifestOverwrite, paths_equivalent, runtime_manifest_path, runtime_registry_dir, - sort_manifests_newest_install_first, validate_runtime_manifest_for_activation, - write_runtime_manifest, + ADOPTED_RUNTIME_ARTIFACT_URL, ManifestOverwrite, paths_equivalent, runtime_manifest_path, + runtime_registry_dir, sort_manifests_newest_install_first, + validate_runtime_manifest_for_activation, write_runtime_manifest, }; use crate::therock; @@ -298,7 +298,7 @@ pub(crate) fn adopt_runtime_from_probe( family_source: "runtime_id".to_owned(), version, install_root: install_root.clone(), - selected_artifact_url: "adopted-read-only".to_owned(), + selected_artifact_url: ADOPTED_RUNTIME_ARTIFACT_URL.to_owned(), source_layout_generation: None, index_url: None, tarball_file_name: None, @@ -644,12 +644,36 @@ mod tests { Ok(()) } - /// The other half of the same derivation: an adopted environment that does - /// expose a CMake path has the toolchain, and the manifest must say so or - /// the next `rocm update` would quietly reinstall without it. - #[test] - fn runtime_adopt_records_devel_when_the_probe_finds_cmake() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt-devel"); + /// Adopts a fake external TheRock venv under `root` whose probe reports a + /// CMake path, through the same `adopt_runtime_from_probe` that + /// `rocm runtimes adopt` runs, and returns the manifest it registered. + fn adopt_test_runtime( + root: &Path, + paths: &AppPaths, + ) -> Result { + adopt_test_runtime_with(root, paths, TestAdoptRequest::HandBuilt) + } + + /// How [`adopt_test_runtime_with`] builds its adoption request. + enum TestAdoptRequest { + /// A request with the runtime id and key spelled out. + HandBuilt, + /// The request `rocm runtimes adopt` builds, through + /// [`infer_adopt_runtime_request`]; `runtime_id` is `--runtime-id` and + /// `channel` is `--channel`. + Inferred { + runtime_id: Option<&'static str>, + channel: Option<&'static str>, + }, + } + + /// Adopts a fake external environment, building the request as `request` + /// says. + fn adopt_test_runtime_with( + root: &Path, + paths: &AppPaths, + request: TestAdoptRequest, + ) -> Result { let external_root = root.join("external-therock-venv"); let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); let python_executable = scripts_dir.join(if cfg!(windows) { @@ -680,38 +704,59 @@ mod tests { fs::write(&amdhip, "amdhip")?; fs::write(&hipblas, "hipblas")?; - let adopted = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { + let probe = therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + cmake_path: Some(cmake_path), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }; + let request = match request { + TestAdoptRequest::HandBuilt => AdoptRuntimeRequest { python_executable, install_root: external_root, runtime_id: "therock-release:gfx120X-all".to_owned(), runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), replace: false, }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - cmake_path: Some(cmake_path), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() - }, - )?; + TestAdoptRequest::Inferred { + runtime_id, + channel, + } => infer_adopt_runtime_request( + python_executable, + Some(external_root), + runtime_id.map(str::to_owned), + None, + channel.map(str::to_owned), + false, + &probe, + )?, + }; + adopt_runtime_from_probe(paths, request, probe) + } + + /// The other half of the same derivation: an adopted environment that does + /// expose a CMake path has the toolchain, and the manifest must say so or + /// the next `rocm update` would quietly reinstall without it. + #[test] + fn runtime_adopt_records_devel_when_the_probe_finds_cmake() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt-devel"); + let adopted = adopt_test_runtime(&root, &paths)?; assert!( adopted.devel, @@ -722,6 +767,49 @@ mod tests { Ok(()) } + /// Adoption installs nothing, so there is no package source to name: the + /// manifest's artifact URL holds only the adoption marker. The listing must + /// say the source was not recorded rather than present that marker as one. + /// The channel is the one adoption recorded, for each way the request + /// `rocm runtimes adopt` builds can name it: adoption's default, `release`, + /// when neither `--channel` nor `--runtime-id` names one; `nightly` after + /// `--channel nightly`; and `nightly` after `--runtime-id + /// therock-nightly:` with no `--channel`, where the channel comes + /// from the runtime id alone. The family is the one the test probe reports. + #[test] + fn adopted_runtime_lists_its_channel_and_its_source_as_unrecorded() -> Result<()> { + for (runtime_id_flag, channel_flag, expected_channel) in [ + (None, None, "release"), + (None, Some("nightly"), "nightly"), + (Some("therock-nightly:gfx120X-all"), None, "nightly"), + ] { + let flags = format!("--runtime-id {runtime_id_flag:?} --channel {channel_flag:?}"); + let (root, paths) = test_paths("runtime-adopt-provenance"); + let adopted = adopt_test_runtime_with( + &root, + &paths, + TestAdoptRequest::Inferred { + runtime_id: runtime_id_flag, + channel: channel_flag, + }, + )?; + let rendered = + crate::runtimes_cmd::render_runtimes_text(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + + assert_eq!(adopted.channel, expected_channel, "{flags}"); + assert_eq!( + crate::runtimes_cmd::tests::provenance_line(&rendered, &adopted.runtime_key), + format!( + " provenance: channel={expected_channel} source=unrecorded \ + source_layout_generation=unrecorded device_target=unrecorded" + ), + "{flags}:\n{rendered}" + ); + } + Ok(()) + } + #[test] fn runtime_adopt_request_infers_ids_from_probe() -> Result<()> { let (root, _) = test_paths("runtime-adopt-infer"); diff --git a/apps/rocm/src/runtime_install_finalize.rs b/apps/rocm/src/runtime_install_finalize.rs index 04c99b568f..d3409186c9 100644 --- a/apps/rocm/src/runtime_install_finalize.rs +++ b/apps/rocm/src/runtime_install_finalize.rs @@ -193,7 +193,7 @@ pub(crate) fn maybe_auto_install_sdk_preferred_engine( engine, runtime_python_for_key(paths, &finalized.runtime_key).as_deref(), &finalized.runtime_key, - // The engine install failed, so no alignment ran and any + // The engine install failed, so no alignment ran and any torch // divergence found here is a real one. &[], ); @@ -247,20 +247,25 @@ enum EngineDependencyCheck { /// /// `expected` carries any divergence the install made on purpose that shares the /// run with a genuine violation. It is reported separately rather than folded in - /// because the remedy differs: reinstalling the engine repairs `violations` and - /// destroys `expected`, so naming that remedy is only safe while `expected` is - /// empty. + /// because the remedy differs: reinstalling the engine repairs `violations` but + /// would undo a torch divergence, so naming that remedy is only safe while + /// `expected` holds no torch divergence. A tensorizer divergence does not rule + /// it out: the reinstall re-checks tensorizer and upgrades it again. Violated { violations: Vec, expected: Vec, }, /// The only unmet requirements are ones the install deliberately diverged from. /// - /// Torch alignment leaves the runtime holding the SDK's build of the release - /// the engine pins, which the engine's exact-pin metadata cannot express. The - /// distinction matters because the remedy for a real violation — reinstalling - /// the engine — is precisely what would undo the alignment and restore a - /// runtime that cannot open a device. + /// Two installs do this. Torch alignment leaves the runtime holding the SDK's + /// build of the release the engine pins, and the tensorizer upgrade leaves + /// vLLM's runtime holding a newer tensorizer than the one it pins because the + /// pinned one cannot be imported; the engine's exact-pin metadata cannot + /// express either. The distinction matters for torch: the remedy for a real + /// violation — reinstalling the engine — would undo a torch divergence and + /// restore a runtime that cannot open a device. A tensorizer divergence does not + /// rule the reinstall out: the install re-checks tensorizer and upgrades it + /// again. ExpectedDivergence(Vec), /// The check itself could not run (no usable `uv`, unreadable environment). NotVerified(String), @@ -854,27 +859,28 @@ fn engine_dependency_check( /// Each line is judged on its own subject: a genuine violation elsewhere in the /// same run says nothing about the package the install deliberately diverged on, /// and must not drag it along. Lumping the two together loses the distinction -/// exactly where it matters most — the remedy for the genuine violation would -/// reinstall the engine and undo the alignment. +/// exactly where it matters most for torch — the remedy for the genuine violation +/// would reinstall the engine and undo the alignment. A tensorizer divergence is +/// judged by what is installed, not by `diverged`: the reinstall is a fine remedy +/// beside it, because the install upgrades tensorizer again. fn classify_dependency_details(details: Vec, diverged: &[&str]) -> EngineDependencyCheck { if details.is_empty() { return EngineDependencyCheck::Satisfied; } - if diverged.is_empty() { - return EngineDependencyCheck::Violated { - violations: details, - expected: Vec::new(), - }; - } // A line whose subject cannot be parsed is not evidence of a divergence, so it // stays a violation: the conservative side keeps saying something is wrong. + // + // The tensorizer override is judged by what is installed, not by the caller's + // list: the engine install makes it on its own, whether or not torch was + // realigned, and it is a divergence in every one of those runs. let (expected, violations): (Vec, Vec) = details.into_iter().partition(|detail| { - rocm_core::violation_subject(detail).is_some_and(|subject| { - diverged - .iter() - .any(|package| subject.package.eq_ignore_ascii_case(package)) - }) + rocm_core::vllm_tensorizer_override_installed(detail).is_some() + || rocm_core::violation_subject(detail).is_some_and(|subject| { + diverged + .iter() + .any(|package| subject.package.eq_ignore_ascii_case(package)) + }) }); if violations.is_empty() { EngineDependencyCheck::ExpectedDivergence(expected) @@ -886,6 +892,43 @@ fn classify_dependency_details(details: Vec, diverged: &[&str]) -> Engin } } +/// Whether an expected-divergence line is the tensorizer override rather than the +/// torch alignment, which decides which explanation the report points at. +fn is_tensorizer_divergence(detail: &str) -> bool { + rocm_core::vllm_tensorizer_override_installed(detail).is_some() +} + +/// The line that says why a tensorizer divergence is kept, printed with the +/// divergence itself so a later install, which prints no upgrade warning, still +/// carries its own explanation. +/// +/// It names the release the runtime actually holds, read from the same +/// `uv pip check` line the divergence was accepted from, and claims only what +/// that line and the accepted rule establish: vLLM pins an older tensorizer, and +/// the installed one is at or above the fixed release. It does not state the +/// Python version, which the rule does not look at. +fn tensorizer_divergence_note(expected: &[String]) -> Option { + let installed = expected + .iter() + .find_map(|detail| rocm_core::vllm_tensorizer_override_installed(detail))?; + // Parsed from a `uv pip check` line, so flattened like the other values here. + // The override rule only accepts letters, digits and dots, so this cannot change + // the value today; it stays as a guard against a looser rule. + let installed = sanitize_log_value(&installed); + Some(format!( + " note: vLLM pins an older tensorizer release; this runtime has tensorizer {installed}, at or above the fixed release {}, which the vLLM install keeps, so `uv pip check` reports vLLM's pin as unmet", + rocm_core::TENSORIZER_PYTHON314_FIX + )) +} + +/// Whether any expected divergence is something other than the tensorizer +/// upgrade, that is, the torch alignment that the `torch_alignment` block explains. +fn has_torch_divergence(expected: &[String]) -> bool { + expected + .iter() + .any(|detail| !is_tensorizer_divergence(detail)) +} + fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) -> String { let mut output = String::new(); match outcome { @@ -910,19 +953,31 @@ fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) for detail in expected { let _ = writeln!(output, " divergence: {}", sanitize_log_value(detail)); } - if expected.is_empty() { + if let Some(note) = tensorizer_divergence_note(expected) { + let _ = writeln!(output, "{note}"); + } + // A reinstall repairs the violations above. It also re-checks tensorizer + // and upgrades it again, so a tensorizer divergence alone does not make + // it unsafe. A torch divergence does: the reinstall would reinstate the + // engine's own torch build, which cannot open a device against the SDK, + // and naming a remedy that trades one defect for a worse one is not + // worth the convenience; the torch_alignment block above says why. + // + // Proven by e2e scenario `vllm-tensorizer-13` (vllm_tensorizer.feature): + // it starts from a genuine violation beside the upgraded tensorizer that + // only a `--reinstall` clears, takes the command to run from this very + // `action:` line, and checks the violation is gone and tensorizer was + // upgraded again after vLLM's own pin came back. Scenario 12 shows the + // plain repair pass does the same. + if has_torch_divergence(expected) { let _ = writeln!( output, - " action: rocm engines install {engine} --reinstall" + " action: repair the violations above without reinstalling {engine}; a reinstall would undo the divergence kept on purpose (see torch_alignment above)" ); } else { - // The reinstall would repair the violations above and reinstate the - // engine's own build of the diverged package, restoring a runtime - // that cannot open a device. Naming a remedy that trades one defect - // for a worse one is not worth the convenience. let _ = writeln!( output, - " action: repair the violations above without reinstalling {engine}; a reinstall would undo the divergence kept on purpose (see torch_alignment above)" + " action: rocm engines install {engine} --reinstall" ); } } @@ -931,15 +986,28 @@ fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) for detail in details { let _ = writeln!(output, " divergence: {}", sanitize_log_value(detail)); } - // Deliberately not the reinstall remedy. It does not apply to - // either state that reaches here: a reinstall realigns torch again + if let Some(note) = tensorizer_divergence_note(details) { + let _ = writeln!(output, "{note}"); + } + // Deliberately not the reinstall remedy. It does not apply to any + // state that reaches here. For torch, a reinstall realigns it again // and reproduces the SDK build, and where the user has opted out of - // that realignment it would undo the decision they made. Which of - // the two this is, the block above has already said. - let _ = writeln!( - output, - " action: none; this torch stack is kept on purpose (see torch_alignment above)" - ); + // that realignment it would undo the decision they made; which of + // the two this is, the torch_alignment block above has already said. + // For tensorizer, a reinstall is harmless but unnecessary: it puts + // vLLM's own pin back and the install then upgrades tensorizer again, + // and the note printed just above says why the divergence exists. + if has_torch_divergence(details) { + let _ = writeln!( + output, + " action: none; this torch stack is kept on purpose (see torch_alignment above)" + ); + } else { + let _ = writeln!( + output, + " action: none; this tensorizer is kept on purpose (see the note above)" + ); + } } } output @@ -1172,6 +1240,186 @@ pub(crate) mod tests { ); } + const TENSORIZER_UNMET: &str = + "The package `vllm` requires `tensorizer==2.10.1`, but `2.12.1` is installed"; + + #[test] + fn the_tensorizer_upgrade_is_an_expected_divergence_without_any_torch_alignment() { + // The install upgrades vLLM's pinned tensorizer. `diverged` is empty here: + // torch was not touched. + let outcome = classify_dependency_details(vec![TENSORIZER_UNMET.to_owned()], &[]); + + assert!( + matches!(&outcome, EngineDependencyCheck::ExpectedDivergence(d) if d.len() == 1), + "{outcome:?}" + ); + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!(rendered.contains(" dependency_check: expected_divergence\n")); + assert!(rendered.contains("this tensorizer is kept on purpose")); + assert!( + !rendered.contains("torch_alignment") && !rendered.contains("--reinstall"), + "{rendered}" + ); + } + + #[test] + fn the_tensorizer_divergence_explains_itself_without_an_upgrade_warning() { + // A repeat install prints no upgrade warning, so the block carries its own + // reason and the action line points at that note. + let outcome = classify_dependency_details(vec![TENSORIZER_UNMET.to_owned()], &[]); + let rendered = render_engine_dependency_check("vllm", &outcome); + + assert!( + rendered.contains(" note: vLLM pins an older tensorizer release; this runtime has tensorizer 2.12.1, at or above the fixed release 2.12.1, which the vLLM install keeps, so `uv pip check` reports vLLM's pin as unmet"), + "{rendered}" + ); + assert!( + rendered.contains( + " action: none; this tensorizer is kept on purpose (see the note above)\n" + ), + "{rendered}" + ); + } + + #[test] + fn the_note_names_the_tensorizer_the_runtime_actually_holds() { + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `tensorizer==2.10.1`, but `2.13.0` is installed" + .to_owned(), + ], + &[], + ); + + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!( + rendered.contains( + "this runtime has tensorizer 2.13.0, at or above the fixed release 2.12.1" + ), + "{rendered}" + ); + } + + #[test] + fn a_version_with_control_characters_is_a_violation_and_is_flattened_when_printed() { + // A local segment may only hold letters, digits and dots, so such a version + // is not the override and no divergence note prints it; it is a plain + // violation, whose line is flattened like every other value here. + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `tensorizer==2.10.1`, but `2.12.1+x\ny` is installed" + .to_owned(), + ], + &[], + ); + + assert!( + matches!(outcome, EngineDependencyCheck::Violated { .. }), + "{outcome:?}" + ); + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!(rendered.contains("`2.12.1+x y` is installed"), "{rendered}"); + assert!(!rendered.contains("\ny"), "{rendered}"); + } + + #[test] + fn another_packages_tensorizer_pin_is_not_the_vllm_override() { + let outcome = classify_dependency_details( + vec![ + "The package `lemonade` requires `tensorizer==2.10.1`, but `2.12.1` is installed" + .to_owned(), + ], + &[], + ); + + assert!( + matches!(outcome, EngineDependencyCheck::Violated { .. }), + "{outcome:?}" + ); + } + + #[test] + fn a_torch_and_a_tensorizer_divergence_together_keep_both_explanations() { + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed".to_owned(), + TENSORIZER_UNMET.to_owned(), + ], + &["torch"], + ); + + assert!( + matches!(&outcome, EngineDependencyCheck::ExpectedDivergence(d) if d.len() == 2), + "{outcome:?}" + ); + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!( + rendered.contains(" note: vLLM pins an older tensorizer release"), + "{rendered}" + ); + assert!( + rendered.contains( + "action: none; this torch stack is kept on purpose (see torch_alignment above)" + ), + "{rendered}" + ); + } + + #[test] + fn a_pin_on_a_newer_tensorizer_than_the_one_installed_is_still_a_violation() { + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `tensorizer==2.13.0`, but `2.12.1` is installed" + .to_owned(), + ], + &[], + ); + + assert!( + matches!(outcome, EngineDependencyCheck::Violated { .. }), + "{outcome:?}" + ); + } + + #[test] + fn a_tensorizer_older_than_the_fix_is_still_a_violation() { + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `tensorizer==2.10.1`, but `2.9.0` is installed" + .to_owned(), + ], + &[], + ); + + assert!( + matches!(outcome, EngineDependencyCheck::Violated { .. }), + "{outcome:?}" + ); + } + + #[test] + fn a_genuine_violation_next_to_the_tensorizer_upgrade_advises_the_reinstall_that_keeps_it() { + // A reinstall repairs numpy and re-checks tensorizer, which it upgrades + // again, so unlike a torch divergence it is the right remedy here. + let outcome = classify_dependency_details( + vec![ + TENSORIZER_UNMET.to_owned(), + "The package `vllm` requires `numpy==2.0.0`, but `1.26.0` is installed".to_owned(), + ], + &[], + ); + + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!(rendered.contains(" dependency_check: violated\n")); + assert!(rendered.contains(" violation: The package `vllm` requires `numpy")); + assert!(rendered.contains(" divergence: The package `vllm` requires `tensorizer")); + assert!(rendered.contains(" note: vLLM pins an older tensorizer release")); + assert!( + rendered.contains(" action: rocm engines install vllm --reinstall\n"), + "{rendered}" + ); + } + #[test] fn a_divergence_on_any_other_package_is_still_a_violation() { // The unrelated violation is real and must keep saying so, but it does not diff --git a/apps/rocm/src/runtime_install_finalize/torch_alignment.rs b/apps/rocm/src/runtime_install_finalize/torch_alignment.rs index d604a44e66..2fbcabee59 100644 --- a/apps/rocm/src/runtime_install_finalize/torch_alignment.rs +++ b/apps/rocm/src/runtime_install_finalize/torch_alignment.rs @@ -1307,8 +1307,10 @@ mod tests { "opted out, all four are the user's: {opted_out:?}" ); - // Not opted out, only torch is ever an intended divergence: the siblings - // are real violations, because rocm-cli realigns torch and nothing else. + // Not opted out, torch is the only intended divergence of the torch stack: + // its siblings are real violations, because rocm-cli realigns torch and + // nothing else of that stack. (vLLM's own unmet tensorizer pin is judged + // separately, by what is installed; this fixture has no such line.) let aligned = classify_dependency_details( details, &diverged_packages(retained_diverged_package(&kept), false), diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 41eabb776e..ac6392d7b8 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -18,9 +18,13 @@ //! also holds `paths_equivalent` and `normalize_path_for_compare`, which adopt //! and uninstall both need, and the on-disk active-runtime marker //! (`ActiveRuntimeMarker`, `active_runtime_marker_path`, -//! `active_runtime_marker_matches`) that activation writes and uninstall reads -//! and clears. The serde shape of [`InstalledRuntimeManifest`] and its probe -//! structs is the on-disk contract of the registry files and of each install's +//! `active_runtime_marker_matches`, `write_active_runtime_marker`, +//! `forget_active_runtime_rollback_target`) that activation and rollback write, +//! and that uninstall reads, removes when it names the removed runtime as the +//! one in use, and otherwise rewrites to drop that runtime as its rollback +//! target; `runtimes_cmd.rs`'s module doc lists every writer and remover. The +//! serde shape of [`InstalledRuntimeManifest`] and its probe structs is the +//! on-disk contract of the registry files and of each install's //! `.rocm-cli-runtime.json`; the marker is user state too, and its serde shape //! is as fixed. @@ -108,6 +112,18 @@ const fn devel_default_for_legacy_manifest() -> bool { true } +/// The `selected_artifact_url` an adopted runtime records (`rocm runtimes +/// adopt`). Adoption installs nothing, so there is no artifact to name; this +/// marks that instead, and is never a package source. +pub(crate) const ADOPTED_RUNTIME_ARTIFACT_URL: &str = "adopted-read-only"; + +/// The value a provenance chain chose, trimmed, or `None` when there is none or +/// it is blank. Applied once, after the chain has picked its field: a blank +/// value is shown as `unrecorded`, it does not make the chain try the next one. +fn recorded(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + impl InstalledRuntimeManifest { /// Whether this runtime has the compiler toolchain. /// @@ -130,6 +146,49 @@ impl InstalledRuntimeManifest { !self.read_only && self.imported_from.is_none() } + /// The source layout generation this runtime recorded: the manifest's own + /// field, else the one its wheel composition was written with. `None` when + /// neither is recorded. + /// + /// Shared by update planning (which treats `None` as the canonical stream) + /// and by [`Self::provenance`], so both choose the same recorded field. + /// They differ in what they do with its value: update planning uses it + /// as recorded, while the listing trims it and prints a blank one (or + /// `None`) as `unrecorded`. + pub(crate) fn recorded_source_layout_generation(&self) -> Option<&str> { + self.source_layout_generation.as_deref().or_else(|| { + self.wheel_composition + .as_ref() + .map(|composition| composition.source_layout_generation.as_str()) + }) + } + + /// Where this runtime came from, as far as its manifest recorded it. + /// + /// The layout and target come from the same recorded field update planning + /// uses, through [`Self::recorded_source_layout_generation`] and + /// [`wheel_composition_device_target`]; the source is the package index, + /// else the artifact URL unless it is the adoption marker. Each chain moves + /// to its next field only when a field is absent. The chosen value then + /// goes through [`recorded`], which trims it and turns a blank one into + /// `None` (shown as `unrecorded`) rather than skipping it. + pub(crate) fn provenance(&self) -> RuntimeProvenance<'_> { + RuntimeProvenance { + channel: recorded(Some(self.channel.as_str())), + // A wheel install records the package index it resolved from; a + // tarball install records only the archive it downloaded. An + // adopted runtime records neither, only the adoption marker. + source: recorded(self.index_url.as_deref().or_else(|| { + Some(self.selected_artifact_url.as_str()) + .filter(|url| *url != ADOPTED_RUNTIME_ARTIFACT_URL) + })), + source_layout_generation: recorded(self.recorded_source_layout_generation()), + device_target: recorded(wheel_composition_device_target( + self.wheel_composition.as_ref(), + )), + } + } + pub(crate) fn normalize_host_paths(mut self) -> Self { self.install_root = normalize_manifest_path(self.install_root); self.python_launcher = self @@ -169,6 +228,55 @@ impl InstalledRuntimeManifest { } } +/// Printed in place of a provenance fact the runtime's manifest does not record. +/// +/// Older manifests predate some of these fields, and a tarball install never +/// had an exact GPU target. [`RuntimeProvenance::fields`] shows this for a fact +/// [`InstalledRuntimeManifest::provenance`] left `None`: its chain of recorded +/// fields found nothing, or found a blank value. Neither ever guesses one fact +/// from another — a target from the family, or a layout from the channel, +/// would read as something the install recorded. +pub(crate) const UNRECORDED_PROVENANCE: &str = "unrecorded"; + +/// Where an installed runtime came from: the channel, the package source it was +/// resolved from, the source layout generation, and the exact GPU target its +/// packages were installed for. Every field is what the manifest recorded, +/// trimmed, and `None` when it recorded nothing or a blank value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RuntimeProvenance<'a> { + pub channel: Option<&'a str>, + pub source: Option<&'a str>, + pub source_layout_generation: Option<&'a str>, + pub device_target: Option<&'a str>, +} + +impl<'a> RuntimeProvenance<'a> { + /// Each fact as `(name, recorded value)`, in display order, with `None` for + /// a fact that was not recorded. + /// + /// The one place the names and their order are written down: + /// [`Self::fields`] shows these, and `rocm runtimes list --json` writes them + /// as its `provenance` object, with `null` for `None`. + pub(crate) const fn recorded_fields(&self) -> [(&'static str, Option<&'a str>); 4] { + [ + ("channel", self.channel), + ("source", self.source), + ("source_layout_generation", self.source_layout_generation), + ("device_target", self.device_target), + ] + } + + /// Each fact as `(name, shown value)`, in display order, with + /// [`UNRECORDED_PROVENANCE`] for a fact that was not recorded. + /// + /// `rocm runtimes list` and `rocm examine` both print from this, so the + /// names and values the two show for one runtime cannot drift apart. + pub(crate) fn fields(&self) -> [(&'static str, &'a str); 4] { + self.recorded_fields() + .map(|(name, value)| (name, value.unwrap_or(UNRECORDED_PROVENANCE))) + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct RocmSdkPythonProbe { #[serde(default)] @@ -665,10 +773,17 @@ pub(crate) fn select_runtime_manifest<'a>( pub(crate) fn runtime_usability_status(manifest: &InstalledRuntimeManifest) -> String { match validate_runtime_manifest_for_activation(manifest) { Ok(()) => "ready".to_owned(), - Err(error) => format!("unusable ({error})"), + Err(error) => unusable_status_text(&error.to_string()), } } +/// How the text reports name a runtime that fails the manifest check +/// activation applies, given the reason it fails it. `rocm runtimes list` +/// prints the same words from the reason it also carries in its JSON. +pub(crate) fn unusable_status_text(reason: &str) -> String { + format!("unusable ({reason})") +} + pub(crate) fn validate_runtime_manifest_for_activation( manifest: &InstalledRuntimeManifest, ) -> Result<()> { @@ -769,6 +884,81 @@ pub(crate) fn active_runtime_marker_path(paths: &AppPaths) -> PathBuf { paths.data_dir.join("runtimes").join("active.json") } +/// Replace the active-runtime marker in one step. +/// +/// Two `rocm` processes can activate a runtime against the same data +/// directory at once (two terminals, a script), so the write goes through the +/// shared atomic writer: each writer stages into its own exclusively created +/// temp file, and the marker is replaced rather than removed first, so a +/// reader always finds a complete marker: on Windows the publish renames first, +/// which replaces in one step. Only when that rename fails, because another +/// process holds the marker open without delete sharing, does it fall back to +/// `ReplaceFileW`, and a `ReplaceFileW` that fails part-way can leave the marker +/// missing, which the writer repairs with a plain rename when that succeeds +/// (see `rocm_core::atomic_write`); two replacements in flight at once through +/// the fallback are not tested. It does not order the two activations: whichever +/// publishes last wins. +pub(crate) fn write_active_runtime_marker( + paths: &AppPaths, + marker: ActiveRuntimeMarker, +) -> Result<()> { + let path = active_runtime_marker_path(paths); + // The shared helper rather than a local temp-file-and-rename: it is the same + // write the failed-activation restore uses, and it is strictly stronger than + // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a + // sharing violation when another process holds the marker open. The helper + // renames first there and falls back to `ReplaceFileW` when that fails. The + // destination is never removed first, on any platform — that would leave a + // window with no marker at all, which runtime resolution and the storage + // retention holds both read as "nothing is active" — and a failed publish + // takes its temp file with it, so a retried activation cannot strand one in + // the folder `rocm runtimes` reads. The one exception is a Windows publish + // that took the old marker away and could not put the new one in place: + // the temp file is then the only copy, so it is kept and the error names + // it; a failed activation removes it again once the restore has put the + // previous marker back (`runtimes_cmd::discard_failed_marker_replacement`). + // It also creates the `runtimes` folder when it is missing. + let bytes = + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; + rocm_core::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write active runtime marker {}", path.display())) +} + +/// Drops `runtime_key` from the active-runtime marker's rollback target, keeping +/// every other field, and returns whether the marker changed. +/// +/// The read-modify-write of the marker for a removed runtime: no marker +/// (nothing, or something other than a file, at its path), one with no rollback +/// target, and one whose rollback target is another runtime are all left alone: +/// it returns `false` and writes nothing. The write is +/// [`write_active_runtime_marker`], the one activation uses. +pub(crate) fn forget_active_runtime_rollback_target( + paths: &AppPaths, + runtime_key: &str, +) -> Result { + let marker_path = active_runtime_marker_path(paths); + // As in `active_runtime_marker_matches`: anything but a file there is no + // marker, so there is no rollback target to drop. + if !marker_path.is_file() { + return Ok(false); + } + let bytes = fs::read(&marker_path) + .with_context(|| format!("failed to read {}", marker_path.display()))?; + let mut marker: ActiveRuntimeMarker = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", marker_path.display()))?; + if !marker + .previous_runtime_key + .as_deref() + .is_some_and(|previous| previous.eq_ignore_ascii_case(runtime_key)) + { + return Ok(false); + } + marker.previous_runtime_key = None; + marker.previous_runtime_id = None; + write_active_runtime_marker(paths, marker)?; + Ok(true) +} + #[cfg(test)] mod tests { use super::*; @@ -828,6 +1018,20 @@ mod tests { } } + /// The adoption marker is persisted manifest format: every runtime adopted + /// so far holds this exact string in its `selected_artifact_url` on disk, + /// and the listing recognises it by comparison. Changing the value without + /// a migration would make those runtimes list the marker as their package + /// source. In-crate code — the producer, the reader and the other unit + /// tests — uses the constant, so it cannot notice the value changing. Two + /// checks pin the on-disk literal instead: this test, and scenario + /// `runtime-lifecycle-22`, whose `plant_adopted_runtime` step writes the + /// literal into a manifest and expects it listed as `source=unrecorded`. + #[test] + fn adoption_marker_keeps_its_persisted_value() { + assert_eq!(ADOPTED_RUNTIME_ARTIFACT_URL, "adopted-read-only"); + } + /// A manifest from before compositions were recorded has only the field, /// and one older still has neither — those installs all had the toolchain. #[test] @@ -1774,4 +1978,90 @@ mod tests { ); fs::remove_dir_all(root).ok(); } + + fn marker_with_previous(previous_key: &str) -> ActiveRuntimeMarker { + ActiveRuntimeMarker { + runtime_id: "therock-nightly:gfx1151".to_owned(), + runtime_key: "nightly-pip-gfx1151-7-14-0a1".to_owned(), + manifest_path: PathBuf::from("registry/nightly-pip-gfx1151-7-14-0a1.json"), + install_root: PathBuf::from("wheel/nightly-pip-gfx1151-7-14-0a1"), + previous_runtime_id: Some("therock-release:gfx1151".to_owned()), + previous_runtime_key: Some(previous_key.to_owned()), + activated_at_unix_ms: 1_700_000_000_000, + } + } + + /// A rollback target that names another runtime is not this uninstall's to + /// drop: the marker is left byte for byte as it was. + #[test] + fn forgetting_a_rollback_target_leaves_a_marker_naming_another_runtime_untouched() -> Result<()> + { + let (root, paths) = test_paths("forget-rollback-other-runtime"); + write_active_runtime_marker(&paths, marker_with_previous("release-pip-gfx1151-7-13-0"))?; + let before = fs::read(active_runtime_marker_path(&paths))?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-12-0")?; + + assert!(!changed); + assert_eq!(fs::read(active_runtime_marker_path(&paths))?, before); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A marker with no rollback target has nothing to drop: it is left byte + /// for byte as it was. + #[test] + fn forgetting_a_rollback_target_leaves_a_marker_without_one_untouched() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-none"); + let mut marker = marker_with_previous("release-pip-gfx1151-7-13-0"); + marker.previous_runtime_id = None; + marker.previous_runtime_key = None; + write_active_runtime_marker(&paths, marker)?; + let before = fs::read(active_runtime_marker_path(&paths))?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-13-0")?; + + assert!(!changed); + assert_eq!(fs::read(active_runtime_marker_path(&paths))?, before); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Runtime keys compare without regard to case everywhere else in the + /// registry, so a case variant of the rollback target is the same runtime. + #[test] + fn forgetting_a_rollback_target_matches_its_key_without_regard_to_case() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-case"); + write_active_runtime_marker(&paths, marker_with_previous("release-pip-gfx1151-7-13-0"))?; + + let changed = forget_active_runtime_rollback_target(&paths, "RELEASE-PIP-GFX1151-7-13-0")?; + + assert!(changed); + let after: ActiveRuntimeMarker = + serde_json::from_slice(&fs::read(active_runtime_marker_path(&paths))?)?; + assert_eq!(after.previous_runtime_key, None); + assert_eq!(after.previous_runtime_id, None); + let in_use = "nightly-pip-gfx1151-7-14-0a1"; + assert_eq!(after.runtime_key, in_use); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// With no marker there is no rollback target to drop, and that is not an + /// error: uninstalling a runtime that was never activated must succeed. It + /// leaves the data folder as it found it: no marker, and not even the + /// `runtimes` folder the marker writer would create. + #[test] + fn forgetting_a_rollback_target_without_a_marker_changes_nothing() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-no-marker"); + fs::create_dir_all(&paths.data_dir)?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-13-0")?; + + assert!(!changed); + assert!(!active_runtime_marker_path(&paths).exists()); + assert_eq!(fs::read_dir(&paths.data_dir)?.count(), 0); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/runtime_uninstall.rs b/apps/rocm/src/runtime_uninstall.rs index e140cf1550..a42f4cb868 100644 --- a/apps/rocm/src/runtime_uninstall.rs +++ b/apps/rocm/src/runtime_uninstall.rs @@ -4,8 +4,9 @@ //! `rocm runtimes uninstall`: planning the removal of a registered runtime, //! re-validating that plan after the confirmation prompt, and applying it -//! (unregistering the runtime, clearing the active and default state that named -//! it, and removing its install root when ROCm CLI owns it). When that root +//! (unregistering the runtime, clearing the active, rollback and default state +//! that named it in both the config and the active-runtime marker, and removing +//! its install root when ROCm CLI owns it). When that root //! holds the Lemonade install its record points at (a self-managed engine //! installs inside the active runtime's folder), the plan says so and applying //! it clears the record, so Lemonade does not keep naming deleted files. @@ -14,10 +15,11 @@ //! `runtimes_cmd.rs` calls into this module from `runtimes()` and re-exports //! `uninstall_runtime`, `should_remove_runtime_install_root`, //! `InstallRootDecision` and `lemonade_record_report_line` for `storage.rs`. -//! The active-runtime marker check this module makes -//! (`active_runtime_marker_matches`, over the `ActiveRuntimeMarker` the -//! activation writes) and the path comparison it needs live in -//! `runtime_registry.rs`, so this module does not depend on `runtimes_cmd.rs`. +//! The active-runtime marker check and update this module makes +//! (`active_runtime_marker_matches` and `forget_active_runtime_rollback_target`, +//! which writes through the same `write_active_runtime_marker` activation uses) +//! and the path comparison it needs live in `runtime_registry.rs`, so this +//! module does not depend on `runtimes_cmd.rs`. use std::fs; use std::path::{Path, PathBuf}; @@ -29,7 +31,8 @@ use rocm_engine_lemonade::RecordCleanup; use crate::cli_report; use crate::runtime_registry::{ active_runtime_marker_matches, active_runtime_marker_path, current_runtime_manifest, - paths_equivalent, runtime_manifest_path, select_runtime_manifest, + forget_active_runtime_rollback_target, paths_equivalent, runtime_manifest_path, + select_runtime_manifest, }; use crate::therock; @@ -360,6 +363,10 @@ pub(crate) fn apply_runtime_uninstall( ) })?; } + } else { + // The marker keeps naming the runtime in use; only a rollback target + // that names the removed runtime is dropped, as the config already did. + forget_active_runtime_rollback_target(paths, &manifest.runtime_key)?; } // Decided from the folder actually removed, not from the plan: the record @@ -505,6 +512,149 @@ mod tests { Ok(()) } + /// Removing the rollback target forgets it in both places that record it. + /// The config already lost it; the active-runtime marker kept naming it as + /// previous, and storage retention reads that field as a runtime to hold. + /// The marker must still name the runtime in use, unchanged. + #[test] + fn uninstalling_the_rollback_target_clears_it_from_the_active_marker() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-rollback-target-marker"); + let previous = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let active = write_test_pip_runtime( + &paths, + "nightly-pip-gfx1151-7-14-0a1", + "therock-nightly:gfx1151", + "7.14.0a1", + 20, + )?; + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &previous.runtime_key)?; + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &active.runtime_key)?; + let read_marker = || -> Result { + Ok(serde_json::from_slice(&fs::read( + active_runtime_marker_path(&paths), + )?)?) + }; + let before = read_marker()?; + assert_eq!( + before.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + + uninstall_runtime(&paths, &mut config, &previous.runtime_key)?; + + let after = read_marker()?; + assert_eq!(after.previous_runtime_key, None, "{after:?}"); + assert_eq!(after.previous_runtime_id, None, "{after:?}"); + assert_eq!(after.runtime_key, before.runtime_key); + assert_eq!(after.runtime_id, before.runtime_id); + assert_eq!(after.manifest_path, before.manifest_path); + assert_eq!(after.install_root, before.install_root); + assert_eq!(after.activated_at_unix_ms, before.activated_at_unix_ms); + assert_eq!(config.previous_runtime_key, None); + assert_eq!( + config.active_runtime_key.as_deref(), + Some(active.runtime_key.as_str()) + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The config and the marker record the rollback target separately and can + /// disagree: here the config has already let it go while the marker still + /// names it. Uninstalling that runtime must clear it from the marker anyway, + /// leaving the runtime in use as it was. + #[test] + fn uninstall_clears_a_rollback_target_only_the_marker_still_names() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-marker-only-rollback-target"); + let previous = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let active = write_test_pip_runtime( + &paths, + "nightly-pip-gfx1151-7-14-0a1", + "therock-nightly:gfx1151", + "7.14.0a1", + 20, + )?; + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &previous.runtime_key)?; + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &active.runtime_key)?; + assert_eq!( + config.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + // The uninstall reads this in-memory config, so nothing is saved. + config.previous_runtime_key = None; + let read_marker = || -> Result { + Ok(serde_json::from_slice(&fs::read( + active_runtime_marker_path(&paths), + )?)?) + }; + let before = read_marker()?; + assert_eq!( + before.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + assert_eq!(config.previous_runtime_key, None); + + uninstall_runtime(&paths, &mut config, &previous.runtime_key)?; + + let after = read_marker()?; + assert_eq!(after.previous_runtime_key, None, "{after:?}"); + assert_eq!(after.previous_runtime_id, None, "{after:?}"); + assert_eq!(after.runtime_key, before.runtime_key); + assert_eq!(after.runtime_id, before.runtime_id); + assert_eq!(after.manifest_path, before.manifest_path); + assert_eq!(after.install_root, before.install_root); + assert_eq!(after.activated_at_unix_ms, before.activated_at_unix_ms); + assert_eq!( + config.active_runtime_key.as_deref(), + Some(active.runtime_key.as_str()) + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Something other than a file at the marker path is not a marker: the + /// active-runtime check reads it as no marker, and so must the rollback-target + /// step, or uninstalling an unrelated runtime fails after its folder and + /// registry entry are already gone. + #[test] + fn uninstall_treats_a_folder_at_the_marker_path_as_no_marker() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-marker-path-is-a-folder"); + let runtime = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let marker_path = active_runtime_marker_path(&paths); + fs::create_dir_all(&marker_path)?; + let mut config = RocmCliConfig::default(); + + uninstall_runtime(&paths, &mut config, &runtime.runtime_key)?; + + assert!(!runtime.install_root.exists()); + assert!(!runtime_manifest_path(&paths, &runtime.runtime_key).exists()); + assert!(marker_path.is_dir()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + /// A record that cannot be read may well name the runtime being removed, so /// the uninstall cannot tell whether it is left dangling. When the runtime /// held a Lemonade tree it must say so rather than treat the record as diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 950960a6b2..b42621f43a 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -3,28 +3,37 @@ // SPDX-License-Identifier: MIT //! `rocm runtimes` command: its dispatch, activate/rollback, and the runtime -//! list rendering. Import and adopt (and recovering the runtime `rocm setup` -//! created) live in `runtime_adopt.rs`, and uninstall in +//! list, which `runtimes_listing` computes once into a `RuntimesListing` that +//! prints either as the text table or, with `--json`, as that value serialized +//! (`docs/json-output.md`). Import and adopt (and recovering the runtime +//! `rocm setup` created) live in `runtime_adopt.rs`, and uninstall in //! `runtime_uninstall.rs`. Registry selection and manifest validation live in //! `runtime_registry.rs`; the names other modules import from here //! (`current_runtime_manifest`, `select_runtime_manifest`, //! `validate_runtime_manifest_for_activation`, `runtime_usability_status`, //! and the registry path helpers) are re-exported from this module. //! -//! Mechanically relocated from `main.rs` with no behavior change — the -//! `dispatch()` call site stays byte-identical (`runtimes(command)`, -//! re-imported via `use crate::runtimes_cmd::{runtimes, ...};`). +//! `runtimes(command)` is the entry point `main.rs`'s `dispatch()` calls for +//! `rocm runtimes`, imported there via +//! `use crate::runtimes_cmd::{runtimes, ...};`. //! `RuntimesCommand` lives in `cli.rs` (part of the `Command`/`Cli` clap //! definition) and is reached via `use crate::RuntimesCommand`, a re-import in //! `main.rs`, the same way `engines_cmd.rs` reaches `EnginesCommand`. //! `RuntimeActivationResult` stays //! `pub(crate)`: it is read by `append_update_activate_summary` in //! `update_cmd.rs`'s `apply_runtime_update` flow. The uninstall plan and apply -//! code lives in `runtime_uninstall.rs`, and the active-runtime marker model in -//! `runtime_registry.rs`; `uninstall_runtime`, -//! `should_remove_runtime_install_root`, `InstallRootDecision`, -//! `ActiveRuntimeMarker` and `active_runtime_marker_path` are re-exported from -//! here for `storage.rs` and `main.rs`. +//! code lives in `runtime_uninstall.rs`, and the active-runtime marker model and +//! `write_active_runtime_marker` in `runtime_registry.rs`; activation and +//! rollback here write the marker through it, as uninstall does when it drops a +//! rollback target. A failed activation's restore and three removals touch the +//! marker without it. The restore puts the earlier bytes back with +//! `rocm_core::atomic_write::write_file_atomically`, or removes the marker when +//! there was none before. Uninstalling the runtime in use removes it, and +//! `rocm config set-default-runtime` and `rocm config clear-default-runtime` +//! remove it and ignore a failed removal. +//! `uninstall_runtime`, `should_remove_runtime_install_root`, +//! `InstallRootDecision`, `ActiveRuntimeMarker` and `active_runtime_marker_path` +//! are re-exported from here for `storage.rs` and `main.rs`. //! `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` //! and `setup_runtime_paths` (from `runtime_adopt.rs`) and //! `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported @@ -38,9 +47,11 @@ use std::fmt::Write as _; use std::fs; +use std::path::PathBuf; use anyhow::{Context, Result}; use rocm_core::{AppPaths, RocmCliConfig, require_yes_or_interactive}; +use serde::Serialize; use crate::runtime_adopt::{ AdoptRuntimeOptions, adopt_runtime_from_python_options, import_runtime_manifest, @@ -54,6 +65,9 @@ pub(crate) use crate::runtime_registry::{ runtime_manifest_path, runtime_registry_dir, runtime_usability_status, select_runtime_manifest, validate_runtime_manifest_for_activation, }; +use crate::runtime_registry::{ + RuntimeProvenance, unusable_status_text, write_active_runtime_marker, +}; pub(crate) use crate::runtime_uninstall::{ InstallRootDecision, lemonade_record_report_line, should_remove_runtime_install_root, uninstall_runtime, @@ -86,7 +100,7 @@ pub(crate) fn ensure_runtime_service_restart_approved(command: &RuntimesCommand) restart_services, yes, } => runtime_services::ensure_service_restart_approved(*restart_services, *yes, "rollback"), - RuntimesCommand::List + RuntimesCommand::List { .. } | RuntimesCommand::Uninstall { .. } | RuntimesCommand::Import { .. } | RuntimesCommand::Adopt { .. } => Ok(()), @@ -96,15 +110,15 @@ pub(crate) fn ensure_runtime_service_restart_approved(command: &RuntimesCommand) pub(crate) fn runtimes(command: Option) -> Result<()> { let paths = AppPaths::discover()?; let mut config = RocmCliConfig::load(&paths)?; - let command = command.unwrap_or(RuntimesCommand::List); + let command = command.unwrap_or(RuntimesCommand::List { json: false }); // Checked before anything is written: a refused restart must leave the // previously active runtime in place, not switch the runtime and then // decline the half of the job the user actually asked for. ensure_runtime_service_restart_approved(&command)?; match command { - RuntimesCommand::List => { - print!("{}", render_runtimes_text(&paths, &config)?); + RuntimesCommand::List { json } => { + cli_report::print_json_or_text(json, &runtimes_listing(&paths, &config)?)?; } RuntimesCommand::Activate { runtime, @@ -580,141 +594,497 @@ const ACTIVE_RUNTIME_MARKER: &str = "*"; /// [`ACTIVE_RUNTIME_MARKER`]. const ROLLBACK_RUNTIME_MARKER: &str = "-"; -pub(crate) fn render_runtimes_text(paths: &AppPaths, config: &RocmCliConfig) -> Result { +/// What `rocm runtimes list` reports, computed once and printed either as text +/// or, with `--json`, as this struct serialized. +/// +/// Every per-runtime fact the text shows is a field here, so the text marker +/// and the JSON `active`/`rollback_target` flags come from the same +/// computation. The text adds only presentation: the marker legend, the +/// next-step hint for an empty registry, the build date it derives from +/// `version`, the `unusable (…)` wording around `unusable_reason`, and the word +/// `unrecorded` for a provenance fact the JSON writes as `null`. The JSON +/// form is additive-only: fields may be added, but none is renamed, removed or +/// given a new meaning (see `docs/json-output.md`). +#[derive(Debug, Serialize)] +pub(crate) struct RuntimesListing { + /// `default_runtime_id` from the config, as configured; `null` when unset. + /// Not necessarily the active runtime's `runtime_id`: an `active_runtime_key` + /// that names a runtime takes precedence, and the id may differ in letter + /// case or match nothing. The text form labels it `active_runtime_id:`, as + /// `rocm examine` does. + default_runtime_id: Option, + active_runtime_key: Option, + previous_runtime_key: Option, + /// Paths are written with `to_string_lossy`, so a path that is not valid + /// UTF-8 is listed with U+FFFD in place of its bad bytes rather than + /// failing the whole document. + #[serde(serialize_with = "serialize_path_lossy")] + registry: PathBuf, + #[serde(serialize_with = "serialize_path_lossy")] + marker: PathBuf, + /// What is wrong with the configured active runtime setting, or `null`. + /// See [`active_runtime_problem`] for when it is set beside an `active` + /// runtime. + active_problem: Option, + /// Newest install first, ties broken by `runtime_key`, as the registry + /// loader sorts them (`sort_manifests_newest_install_first`). + runtimes: Vec, +} + +/// What is wrong with the configured active runtime setting. The text form +/// prints it as the `active_status:` line. +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum ActiveRuntimeProblem { + /// `active_runtime_key` is set but names no registered runtime, ASCII + /// letter case ignored. Reported even when `default_runtime_id` then + /// resolves a runtime, which the listing marks `active`. + MissingManifestForKey { runtime_key: String }, + /// No `active_runtime_key`; `default_runtime_id` matches no manifest's id, + /// ASCII letter case ignored. + MissingManifestForId { runtime_id: String }, + /// No `active_runtime_key`; `default_runtime_id` matches more than one + /// manifest's id (letter case ignored), so none is active until one key is + /// activated. + AmbiguousRuntimeId { + runtime_id: String, + runtime_keys: Vec, + }, +} + +/// One registered runtime in [`RuntimesListing`]. +#[derive(Debug, Serialize)] +struct RuntimeListingEntry { + runtime_key: String, + runtime_id: String, + /// The version the manifest records; the text form adds the build date. + version: String, + format: String, + family: String, + mode: RuntimeMode, + /// Serialized as the two JSON fields `status` and `unusable_reason`. + #[serde(flatten)] + usability: RuntimeUsability, + toolchain: RuntimeToolchain, + /// The runtime `current_runtime_manifest` resolves for this config: the + /// one `rocm examine`, `rocm runtimes activate` and `rollback` treat as + /// the active runtime. + active: bool, + /// The runtime `select_runtime_manifest` resolves `previous_runtime_key` + /// to, which is the runtime `rocm runtimes rollback` selects (and then + /// refuses if it is unusable). + rollback_target: bool, + #[serde(serialize_with = "serialize_path_lossy")] + install_root: PathBuf, + /// Where the runtime came from, as its manifest recorded it + /// (`InstalledRuntimeManifest::provenance`). The text form prints it as the + /// `provenance:` line under `install_root:`; the JSON form as an object + /// with the same names, `null` where the text prints `unrecorded`. + provenance: RuntimeListingProvenance, +} + +/// An owned copy of a runtime's [`RuntimeProvenance`], so the listing can +/// outlive the manifests it was computed from. It holds the recorded values +/// (trimmed, `None` when unrecorded). The text prints them through +/// [`RuntimeProvenance::fields`], the same as `rocm examine`, and the JSON +/// writes them through [`RuntimeProvenance::recorded_fields`], so the two forms +/// take their names and order from one list. +#[derive(Debug, Clone, PartialEq, Eq)] +struct RuntimeListingProvenance { + channel: Option, + source: Option, + source_layout_generation: Option, + device_target: Option, +} + +impl RuntimeListingProvenance { + fn recorded(provenance: &RuntimeProvenance<'_>) -> Self { + Self { + channel: provenance.channel.map(str::to_owned), + source: provenance.source.map(str::to_owned), + source_layout_generation: provenance.source_layout_generation.map(str::to_owned), + device_target: provenance.device_target.map(str::to_owned), + } + } + + fn as_provenance(&self) -> RuntimeProvenance<'_> { + RuntimeProvenance { + channel: self.channel.as_deref(), + source: self.source.as_deref(), + source_layout_generation: self.source_layout_generation.as_deref(), + device_target: self.device_target.as_deref(), + } + } +} + +/// One JSON object holding each fact under the name the text prints it with, +/// as a string, or `null` when it was not recorded. Not the text's +/// `unrecorded` word: a script must not read that placeholder as a value. +impl Serialize for RuntimeListingProvenance { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeMap as _; + let fields = self.as_provenance().recorded_fields(); + let mut object = serializer.serialize_map(Some(fields.len()))?; + for (name, value) in fields { + object.serialize_entry(name, &value)?; + } + object.end() + } +} + +/// A path as a JSON string, through `to_string_lossy`. serde's own `Path` +/// impl fails on a path that is not valid UTF-8, which would leave +/// `rocm runtimes list --json` printing nothing for a listing the text form +/// prints (with the same replacement, through `Path::display`). +fn serialize_path_lossy( + path: &std::path::Path, + serializer: S, +) -> Result { + serializer.serialize_str(&path.to_string_lossy()) +} + +/// Whether a runtime carries the compiler toolchain. The text and the JSON +/// print the same word, the one `toolchain_state_text` gives `rocm examine`. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +enum RuntimeToolchain { + Included, + Excluded, +} + +impl RuntimeToolchain { + const fn from_includes_devel(includes_devel: bool) -> Self { + if includes_devel { + Self::Included + } else { + Self::Excluded + } + } + + const fn as_str(self) -> &'static str { + toolchain_state_text(matches!(self, Self::Included)) + } +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum RuntimeMode { + /// ROCm CLI installed it and may remove its folder. + Managed, + /// Imported or adopted; ROCm CLI never changes its folder. + ReadOnly, +} + +impl RuntimeMode { + const fn as_str(self) -> &'static str { + match self { + Self::Managed => "managed", + Self::ReadOnly => "read-only", + } + } +} + +/// Whether a runtime passes the manifest check that `rocm runtimes activate` +/// applies (`validate_runtime_manifest_for_activation`). Passing it does not +/// promise the activation succeeds: that can still refuse on an ambiguous +/// selector, unreadable server records or a failed config or marker write. An +/// unusable runtime always carries the reason, so the listing cannot report one +/// without saying why. +#[derive(Debug)] +enum RuntimeUsability { + Ready, + /// The human-readable message the manifest check failed with (not a + /// stable code). + Unusable { + reason: String, + }, +} + +impl RuntimeUsability { + /// The JSON `status` value. + const fn status(&self) -> &'static str { + match self { + Self::Ready => "ready", + Self::Unusable { .. } => "unusable", + } + } + + /// The JSON `unusable_reason` value: `None` exactly when ready. + fn unusable_reason(&self) -> Option<&str> { + match self { + Self::Ready => None, + Self::Unusable { reason } => Some(reason), + } + } +} + +/// Two fields, `status` (`ready` or `unusable`) and `unusable_reason` (`null` +/// when ready), flattened into each runtime's JSON object. +impl Serialize for RuntimeUsability { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeStruct as _; + let mut fields = serializer.serialize_struct("RuntimeUsability", 2)?; + fields.serialize_field("status", self.status())?; + fields.serialize_field("unusable_reason", &self.unusable_reason())?; + fields.end() + } +} + +/// Recover the runtime `rocm setup` created, load the registry, and compute +/// everything `rocm runtimes list` reports, in either form. +pub(crate) fn runtimes_listing( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result { recover_setup_runtime_registration(paths, config)?; let manifests = therock::load_runtime_manifests(paths)?; - let mut output = String::new(); - let _ = writeln!(output, "registered ROCm runtimes"); - let _ = writeln!( - output, - " active_runtime_id: {}", - config.default_runtime_id.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " active_runtime_key: {}", - config.active_runtime_key.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " previous_runtime_key: {}", - config.previous_runtime_key.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " registry: {}", - runtime_registry_dir(paths).display() - ); - let _ = writeln!( - output, - " marker: {}", - active_runtime_marker_path(paths).display() - ); - if let Some(active_runtime_key) = config.active_runtime_key.as_deref() - && !manifests + Ok(listing_from_manifests( + config, + manifests, + runtime_registry_dir(paths), + active_runtime_marker_path(paths), + )) +} + +/// The listing of `manifests`, in the order given, for `config`. Split from +/// the registry read so every configuration can be tested in memory, including +/// keys that differ only in letter case, which one registry folder cannot hold +/// on a case-insensitive filesystem. +/// +/// `active` and `rollback_target` are not decided by rules of the listing's +/// own: they are the runtimes the commands themselves resolve, by the same +/// functions, so the listing marks what those commands act on. +fn listing_from_manifests( + config: &RocmCliConfig, + manifests: Vec, + registry: PathBuf, + marker: PathBuf, +) -> RuntimesListing { + // Compared by position, not by key: keys can differ only in letter case, + // and the resolver's choice between such twins is part of what is listed. + let position = |resolved: &therock::InstalledRuntimeManifest| { + manifests .iter() - .any(|manifest| manifest.runtime_key == active_runtime_key) - { + .position(|manifest| std::ptr::eq(manifest, resolved)) + }; + let active_index = current_runtime_manifest(config, &manifests).and_then(position); + // `rocm runtimes rollback` selects `previous_runtime_key` this way; a value + // it refuses (no match, or ambiguous) names no rollback target. + let rollback_index = config + .previous_runtime_key + .as_deref() + .and_then(|selector| select_runtime_manifest(&manifests, selector).ok()) + .and_then(position); + let active_problem = active_runtime_problem(config, &manifests); + + let runtimes = manifests + .into_iter() + .enumerate() + .map(|(index, manifest)| { + // The manifest check `rocm runtimes activate` applies, so "ready" + // here means the runtime passes that check, not that an activation + // is sure to succeed (see `RuntimeUsability`). + let usability = match validate_runtime_manifest_for_activation(&manifest) { + Ok(()) => RuntimeUsability::Ready, + Err(error) => RuntimeUsability::Unusable { + reason: error.to_string(), + }, + }; + let mode = if manifest.read_only { + RuntimeMode::ReadOnly + } else { + RuntimeMode::Managed + }; + // The compiler is opt-in and `rocm update` reinstalls whatever this + // says, so an install missing it should not be silent about that. + // `rocm examine` reports the same thing for the active runtime. + let toolchain = RuntimeToolchain::from_includes_devel(manifest.includes_devel()); + let provenance = RuntimeListingProvenance::recorded(&manifest.provenance()); + RuntimeListingEntry { + runtime_key: manifest.runtime_key, + runtime_id: manifest.runtime_id, + version: manifest.version, + format: manifest.format, + family: manifest.family, + mode, + usability, + toolchain, + active: active_index == Some(index), + rollback_target: rollback_index == Some(index), + install_root: manifest.install_root, + provenance, + } + }) + .collect(); + + RuntimesListing { + default_runtime_id: config.default_runtime_id.clone(), + active_runtime_key: config.active_runtime_key.clone(), + previous_runtime_key: config.previous_runtime_key.clone(), + registry, + marker, + active_problem, + runtimes, + } +} + +/// What is wrong with the configured active runtime setting, by the matching +/// `current_runtime_manifest` applies. +/// +/// It reports the configuration, so it can disagree with `active` in exactly +/// one case: an `active_runtime_key` that names no runtime is reported as +/// `MissingManifestForKey` even when the resolver then falls back to +/// `default_runtime_id` and finds one, which is marked active. The commands act +/// on that runtime, so the listing marks it; the key is still dead, so the +/// listing says so. With no `active_runtime_key`, the problem comes from the +/// same `default_runtime_id_matches` the resolver falls back to, so it is set +/// exactly when nothing is active. +fn active_runtime_problem( + config: &RocmCliConfig, + manifests: &[therock::InstalledRuntimeManifest], +) -> Option { + if let Some(runtime_key) = config.active_runtime_key.as_deref() { + // The resolver's key step on its own: the same call with the default + // id it would fall back to taken away. + let mut key_only = config.clone(); + key_only.default_runtime_id = None; + return current_runtime_manifest(&key_only, manifests) + .is_none() + .then(|| ActiveRuntimeProblem::MissingManifestForKey { + runtime_key: runtime_key.to_owned(), + }); + } + let runtime_id = config.default_runtime_id.as_deref()?; + match default_runtime_id_matches(config, manifests).as_slice() { + [] => Some(ActiveRuntimeProblem::MissingManifestForId { + runtime_id: runtime_id.to_owned(), + }), + [_] => None, + matches => Some(ActiveRuntimeProblem::AmbiguousRuntimeId { + runtime_id: runtime_id.to_owned(), + runtime_keys: matches + .iter() + .map(|manifest| manifest.runtime_key.clone()) + .collect(), + }), + } +} + +impl cli_report::JsonOrText for RuntimesListing { + fn render_text(&self) -> String { + let mut output = String::new(); + let unset = |value: &Option| value.clone().unwrap_or_else(|| "".to_owned()); + let _ = writeln!(output, "registered ROCm runtimes"); let _ = writeln!( output, - " active_status: missing manifest for active_runtime_key={active_runtime_key}" + " active_runtime_id: {}", + unset(&self.default_runtime_id) ); - } - if config.active_runtime_key.is_none() - && let Some(default_runtime_id) = config.default_runtime_id.as_deref() - { - let matches = manifests - .iter() - .filter(|manifest| manifest.runtime_id == default_runtime_id) - .collect::>(); - if matches.is_empty() { - let _ = writeln!( - output, - " active_status: missing manifest for active_runtime_id={default_runtime_id}" - ); - } else if matches.len() > 1 { + let _ = writeln!( + output, + " active_runtime_key: {}", + unset(&self.active_runtime_key) + ); + let _ = writeln!( + output, + " previous_runtime_key: {}", + unset(&self.previous_runtime_key) + ); + let _ = writeln!(output, " registry: {}", self.registry.display()); + let _ = writeln!(output, " marker: {}", self.marker.display()); + match &self.active_problem { + None => {} + Some(ActiveRuntimeProblem::MissingManifestForKey { runtime_key }) => { + let _ = writeln!( + output, + " active_status: missing manifest for active_runtime_key={runtime_key}" + ); + } + Some(ActiveRuntimeProblem::MissingManifestForId { runtime_id }) => { + let _ = writeln!( + output, + " active_status: missing manifest for active_runtime_id={runtime_id}" + ); + } + Some(ActiveRuntimeProblem::AmbiguousRuntimeId { + runtime_id, + runtime_keys, + }) => { + let _ = writeln!( + output, + " active_status: ambiguous runtime_id={runtime_id}; activate one runtime_key: {}", + runtime_keys.join(", ") + ); + } + } + if self.runtimes.is_empty() { + let _ = writeln!(output, " installed: none"); let _ = writeln!( output, - " active_status: ambiguous runtime_id={default_runtime_id}; activate one runtime_key: {}", - runtime_keys_text(&matches) + " next step: rocm install sdk --channel release --format wheel" ); + return output; } - } - if manifests.is_empty() { - let _ = writeln!(output, " installed: none"); - let _ = writeln!( - output, - " next step: rocm install sdk --channel release --format wheel" - ); - return Ok(output); - } - let default_runtime_matches = default_runtime_id_matches(config, &manifests); - let single_default_runtime_key = - if config.active_runtime_key.is_none() && default_runtime_matches.len() == 1 { - Some(default_runtime_matches[0].runtime_key.clone()) - } else { - None - }; - drop(default_runtime_matches); - - let _ = writeln!(output, " installed:"); - let _ = writeln!( - output, - " legend: {ACTIVE_RUNTIME_MARKER} = active, {ROLLBACK_RUNTIME_MARKER} = rollback target" - ); - let _ = writeln!(output); - for manifest in manifests { - let active = config - .active_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key == manifest.runtime_key) - || single_default_runtime_key.as_deref() == Some(manifest.runtime_key.as_str()); - let rollback = config - .previous_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key == manifest.runtime_key); - let marker = if active { - Some(ACTIVE_RUNTIME_MARKER) - } else if rollback { - Some(ROLLBACK_RUNTIME_MARKER) - } else { - None - }; - let status = runtime_usability_status(&manifest); - let mode = if manifest.read_only { - "read-only" - } else { - "managed" - }; - // The compiler is opt-in and `rocm update` reinstalls whatever this - // says, so an install missing it should not be silent about that. - // `rocm examine` reports the same thing for the active runtime. - let toolchain = toolchain_state_text(manifest.includes_devel()); - cli_report::write_marked_row( - &mut output, - " ", - marker, - format_args!( - "{} runtime_id={} version={} format={} family={} mode={} status={} toolchain={}", - manifest.runtime_key, - manifest.runtime_id, - therock::runtime_version_display(&manifest.version), - manifest.format, - manifest.family, - mode, - status, - toolchain - ), - ); + let _ = writeln!(output, " installed:"); let _ = writeln!( output, - " install_root: {}", - manifest.install_root.display() + " legend: {ACTIVE_RUNTIME_MARKER} = active, {ROLLBACK_RUNTIME_MARKER} = rollback target" ); + let _ = writeln!(output); + for runtime in &self.runtimes { + // Active wins when a runtime is both. `rocm runtimes activate` and + // `rollback` never record a runtime as its own rollback target, so + // only a change made outside the CLI reaches this: a hand-edited + // config, or the active runtime's registry entry deleted while the + // default id falls back to the rollback target. + let marker = if runtime.active { + Some(ACTIVE_RUNTIME_MARKER) + } else if runtime.rollback_target { + Some(ROLLBACK_RUNTIME_MARKER) + } else { + None + }; + let status = match &runtime.usability { + RuntimeUsability::Ready => runtime.usability.status().to_owned(), + RuntimeUsability::Unusable { reason } => unusable_status_text(reason), + }; + cli_report::write_marked_row( + &mut output, + " ", + marker, + format_args!( + "{} runtime_id={} version={} format={} family={} mode={} status={} toolchain={}", + runtime.runtime_key, + runtime.runtime_id, + therock::runtime_version_display(&runtime.version), + runtime.format, + runtime.family, + runtime.mode.as_str(), + status, + runtime.toolchain.as_str() + ), + ); + let _ = writeln!( + output, + " install_root: {}", + runtime.install_root.display() + ); + let _ = writeln!( + output, + " provenance: {}", + provenance_text(&runtime.provenance.as_provenance()) + ); + } + output } +} - Ok(output) +/// The text form of `rocm runtimes list`, for callers that want the string +/// (the chat assistant's read-only tool) rather than printing it. +pub(crate) fn render_runtimes_text(paths: &AppPaths, config: &RocmCliConfig) -> Result { + use cli_report::JsonOrText as _; + Ok(runtimes_listing(paths, config)?.render_text()) } pub(crate) fn activate_runtime( @@ -861,6 +1231,18 @@ fn rollback_runtime( }) } +/// A runtime's provenance as the `name=value` pairs `rocm runtimes list` prints +/// on its `provenance:` line. `rocm examine` prints the same pairs for the +/// active runtime as `active_runtime_:` lines. +fn provenance_text(provenance: &RuntimeProvenance<'_>) -> String { + provenance + .fields() + .iter() + .map(|(name, value)| format!("{name}={value}")) + .collect::>() + .join(" ") +} + /// How the CLI names the toolchain state of a runtime. /// /// `rocm runtimes list` reports it per runtime and `rocm examine` reports it @@ -875,43 +1257,6 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { } } -/// Replace the active-runtime marker in one step. -/// -/// Two `rocm` processes can activate a runtime against the same data -/// directory at once (two terminals, a script), so the write goes through the -/// shared atomic writer: each writer stages into its own exclusively created -/// temp file, and the marker is replaced rather than removed first, so a -/// reader always finds a complete marker: on Windows the publish renames first, -/// which replaces in one step. Only when that rename fails, because another -/// process holds the marker open without delete sharing, does it fall back to -/// `ReplaceFileW`, and a `ReplaceFileW` that fails part-way can leave the marker -/// missing, which the writer repairs with a plain rename when that succeeds -/// (see `rocm_core::atomic_write`); two replacements in flight at once through -/// the fallback are not tested. It does not order the two activations: the last -/// one to publish wins, as before. -fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { - let path = active_runtime_marker_path(paths); - // The shared helper rather than a local temp-file-and-rename: it is the same - // write the failed-activation restore uses, and it is strictly stronger than - // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a - // sharing violation when another process holds the marker open. The helper - // renames first there and falls back to `ReplaceFileW` when that fails. The - // destination is never removed first, on any platform — that would leave a - // window with no marker at all, which runtime resolution and the storage - // retention holds both read as "nothing is active" — and a failed publish - // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. The one exception is a Windows publish - // that took the old marker away and could not put the new one in place: - // the temp file is then the only copy, so it is kept and the error names - // it; a failed activation removes it again once the restore has put the - // previous marker back (`discard_failed_marker_replacement`). It also - // creates the `runtimes` folder when it is missing. - let bytes = - serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; - rocm_core::atomic_write::write_file_atomically(&path, &bytes) - .with_context(|| format!("failed to write active runtime marker {}", path.display())) -} - #[cfg(test)] pub(crate) mod tests { use std::path::{Path, PathBuf}; @@ -1211,31 +1556,1836 @@ pub(crate) mod tests { Ok(()) } + /// The row a runtime's `provenance:` line belongs to, found by its key: the + /// first `provenance:` line after that runtime's marked row. + pub(crate) fn provenance_line<'a>(rendered: &'a str, runtime_key: &str) -> &'a str { + let row = format!(" {runtime_key} runtime_id="); + rendered + .lines() + .skip_while(|line| !line.contains(&row)) + .skip(1) + .find(|line| line.trim_start().starts_with("provenance:")) + .unwrap_or_else(|| panic!("no provenance line for {runtime_key}:\n{rendered}")) + } + + /// A nightly ROCm 10 wheel runtime installed for one exact GPU, with every + /// provenance fact recorded the way `rocm install sdk` records it. The + /// family is the family-wide `gfx120X-all`, so a target inferred from the + /// family instead of read from the install shows up as a wrong value. + fn nightly_exact_target_manifest( + base: therock::InstalledRuntimeManifest, + ) -> therock::InstalledRuntimeManifest { + therock::InstalledRuntimeManifest { + runtime_key: "nightly-wheel-gfx1201-10-1-0a20260822".to_owned(), + runtime_id: "therock-nightly:gfx120X-all".to_owned(), + channel: "nightly".to_owned(), + family: "gfx120X-all".to_owned(), + version: "10.1.0a20260822".to_owned(), + selected_artifact_url: "https://example.invalid/whl-multi-arch".to_owned(), + index_url: Some("https://example.invalid/whl-multi-arch".to_owned()), + source_layout_generation: Some("next-v1".to_owned()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "next-v1".to_owned(), + package_specs: vec!["rocm[libraries,device-gfx1201]==10.1.0a20260822".to_owned()], + rocm_sdk_target: Some("gfx1201".to_owned()), + }), + ..base + } + } + + fn rewrite_manifest( + paths: &AppPaths, + manifest: &therock::InstalledRuntimeManifest, + ) -> Result<()> { + fs::write( + runtime_manifest_path(paths, &manifest.runtime_key), + serde_json::to_vec_pretty(manifest)?, + )?; + Ok(()) + } + + /// The install report names the channel, source, layout and exact GPU + /// target; afterwards the only record of them is the manifest, so the + /// listing is where a user finds out which stream and which GPU a runtime + /// is for. Each value is asserted exactly, so a field printed from the + /// wrong source (the family for the target, the channel for the layout) + /// fails here. #[test] - fn runtime_lists_display_build_date_from_version_string() -> Result<()> { - let (root, paths) = test_paths("runtime-build-date-display"); - let runtime_key = "release-pip-gfx120x-all-7-14-0a20260601"; - let manifest = write_test_pip_runtime( + fn runtime_list_names_where_each_runtime_came_from() -> Result<()> { + let (root, paths) = test_paths("runtime-list-provenance"); + let base = write_test_pip_runtime( &paths, - runtime_key, + "release-pip-gfx120x-all-7-13-0", "therock-release:gfx120X-all", - "7.14.0a20260601", + "7.13.0", + 10, + )?; + + // Written before the index, the layout and the composition were + // recorded: only the artifact URL every manifest carries is left. + let legacy = therock::InstalledRuntimeManifest { + index_url: None, + source_layout_generation: None, + wheel_composition: None, + ..base.clone() + }; + rewrite_manifest(&paths, &legacy)?; + + let nightly = nightly_exact_target_manifest(base.clone()); + write_test_pip_runtime( + &paths, + &nightly.runtime_key, + &nightly.runtime_id, + &nightly.version, 20, )?; - let config = RocmCliConfig { - active_runtime_key: Some(manifest.runtime_key.clone()), - default_runtime_id: Some(manifest.runtime_id), - ..RocmCliConfig::default() + rewrite_manifest(&paths, &nightly)?; + + // A tarball install records the archive it downloaded and its layout; + // it was never installed for one exact GPU. + let tarball = therock::InstalledRuntimeManifest { + runtime_key: "release-tarball-gfx120x-all-7-13-0".to_owned(), + format: "tarball".to_owned(), + selected_artifact_url: + "https://example.invalid/tarball/therock-dist-linux-gfx120X-all-7.13.0.tar.gz" + .to_owned(), + index_url: None, + source_layout_generation: Some("multi-arch-v2".to_owned()), + wheel_composition: None, + ..base }; + rewrite_manifest(&paths, &tarball)?; - let runtimes = render_runtimes_text(&paths, &config)?; - assert!(runtimes.contains("version=7.14.0a20260601 (build 2026-06-01)")); + let rendered = render_runtimes_text(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); - let mut examine = String::new(); - append_examine_runtime_state(&mut examine, &paths, &config)?; - assert!(examine.contains("active_runtime_version: 7.14.0a20260601 (build 2026-06-01)")); + // `cargo xtask e2e` prewarm (`assess` in xtask/src/e2e_prewarm.rs) + // reads the line directly after a runtime's row as its + // `install_root:`, so the order row, `install_root:`, `provenance:` is + // pinned here by index; `provenance_line` searches ahead and would not + // notice a reorder. + let lines: Vec<&str> = rendered.lines().collect(); + let row = format!(" {} runtime_id=", nightly.runtime_key); + let row_index = lines + .iter() + .position(|line| line.contains(&row)) + .unwrap_or_else(|| panic!("no row for {}:\n{rendered}", nightly.runtime_key)); + assert_eq!( + lines.get(row_index + 1).copied(), + Some(format!(" install_root: {}", nightly.install_root.display()).as_str()), + "the line after the row must be its install_root:\n{rendered}" + ); + assert_eq!( + lines.get(row_index + 2).copied(), + Some(provenance_line(&rendered, &nightly.runtime_key)), + "the line after install_root: must be its provenance:\n{rendered}" + ); + + assert_eq!( + provenance_line(&rendered, &nightly.runtime_key), + " provenance: channel=nightly source=https://example.invalid/whl-multi-arch \ + source_layout_generation=next-v1 device_target=gfx1201", + "{rendered}" + ); + assert!( + rendered.contains("version=10.1.0a20260822 (build 2026-08-22)"), + "the nightly's version must show its build date:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &legacy.runtime_key), + " provenance: channel=release source=https://example.invalid/therock \ + source_layout_generation=unrecorded device_target=unrecorded", + "{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &tarball.runtime_key), + " provenance: channel=release \ + source=https://example.invalid/tarball/therock-dist-linux-gfx120X-all-7.13.0.tar.gz \ + source_layout_generation=multi-arch-v2 device_target=unrecorded", + "{rendered}" + ); + Ok(()) + } + + /// The layout and target are read from the same recorded fields + /// `rocm update` reads; it does not read the recorded source. Each chain + /// moves to its next field only when a field is absent: package index, + /// else artifact URL (never the adoption marker); layout field, else the + /// wheel composition's layout; `rocm_sdk_target`, else the + /// `device-` package extra. The value chosen is printed trimmed, + /// and a fact recorded nowhere, or recorded blank, reads `unrecorded`. One + /// runtime per rule: + /// + /// - index and artifact URL both recorded and differing: the index wins; + /// - layout field and composition layout differing, and `rocm_sdk_target` + /// and package extra differing: the field earlier in each chain wins; + /// - layout only in the composition (manifests older than the field): it is + /// still the layout; + /// - target only as a package extra (manifests older than + /// `rocm_sdk_target`): it is still the target; + /// - an adopted runtime (marker, no index): source `unrecorded`; + /// - a blank index, layout field and `rocm_sdk_target` in front of a + /// recorded artifact URL, composition layout and package extra: each + /// reads `unrecorded`, since a blank value is not skipped; + /// - an empty `device-` extra and no `rocm_sdk_target`: `unrecorded`; + /// - values padded with whitespace: printed trimmed; + /// - every field blank: each fact `unrecorded`, never an empty `name=`. + /// + /// Blank or padded values are not what `rocm install sdk` writes; they can + /// come from a hand-edited manifest or one brought in with + /// `rocm runtimes import`. + #[test] + fn runtime_list_reads_each_provenance_fact_from_where_it_was_recorded() -> Result<()> { + let (root, paths) = test_paths("runtime-list-provenance-precedence"); + let base = write_test_pip_runtime( + &paths, + "release-pip-index-and-artifact", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + + let index_and_artifact = therock::InstalledRuntimeManifest { + index_url: Some("https://example.invalid/simple".to_owned()), + selected_artifact_url: "https://example.invalid/whl/rocm-7.13.0.tar.gz".to_owned(), + ..base.clone() + }; + rewrite_manifest(&paths, &index_and_artifact)?; + + let composition_layout = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-composition-layout".to_owned(), + source_layout_generation: None, + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "next-v1".to_owned(), + package_specs: vec!["rocm[libraries]==7.13.0".to_owned()], + rocm_sdk_target: None, + }), + ..base.clone() + }; + rewrite_manifest(&paths, &composition_layout)?; + + let spec_target = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-spec-target".to_owned(), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "multi-arch-v2".to_owned(), + package_specs: vec!["rocm[libraries,device-gfx1201]==7.13.0".to_owned()], + rocm_sdk_target: None, + }), + ..base.clone() + }; + rewrite_manifest(&paths, &spec_target)?; + + let adopted = therock::InstalledRuntimeManifest { + runtime_key: "adopted-release-pip".to_owned(), + index_url: None, + selected_artifact_url: crate::runtime_registry::ADOPTED_RUNTIME_ARTIFACT_URL.to_owned(), + read_only: true, + ..base.clone() + }; + rewrite_manifest(&paths, &adopted)?; + + let blank_before_fallback = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-blank-before-fallback".to_owned(), + index_url: Some(String::new()), + selected_artifact_url: "https://example.invalid/therock".to_owned(), + source_layout_generation: Some(String::new()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "next-v1".to_owned(), + package_specs: vec!["rocm[libraries,device-gfx1201]==7.13.0".to_owned()], + rocm_sdk_target: Some(String::new()), + }), + ..base.clone() + }; + rewrite_manifest(&paths, &blank_before_fallback)?; + + let first_in_chain_wins = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-first-in-chain-wins".to_owned(), + source_layout_generation: Some("multi-arch-v2".to_owned()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "next-v1".to_owned(), + package_specs: vec!["rocm[libraries,device-gfx1201]==7.13.0".to_owned()], + rocm_sdk_target: Some("gfx1100".to_owned()), + }), + ..base.clone() + }; + rewrite_manifest(&paths, &first_in_chain_wins)?; + + let padded = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-padded".to_owned(), + channel: " release ".to_owned(), + index_url: Some(" https://example.invalid/simple ".to_owned()), + source_layout_generation: Some(" multi-arch-v2 ".to_owned()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "multi-arch-v2".to_owned(), + package_specs: vec!["rocm[libraries,device-gfx1201]==7.13.0".to_owned()], + rocm_sdk_target: Some(" gfx1201 ".to_owned()), + }), + ..base.clone() + }; + rewrite_manifest(&paths, &padded)?; + + let empty_device_extra = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-empty-device-extra".to_owned(), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "multi-arch-v2".to_owned(), + package_specs: vec!["rocm[libraries,device-]==7.13.0".to_owned()], + rocm_sdk_target: None, + }), + ..base.clone() + }; + rewrite_manifest(&paths, &empty_device_extra)?; + + let all_blank = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-all-blank".to_owned(), + channel: String::new(), + index_url: Some(" ".to_owned()), + selected_artifact_url: String::new(), + source_layout_generation: Some(String::new()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: String::new(), + package_specs: vec!["rocm[libraries]==7.13.0".to_owned()], + rocm_sdk_target: Some(String::new()), + }), + ..base + }; + rewrite_manifest(&paths, &all_blank)?; + let rendered = render_runtimes_text(&paths, &RocmCliConfig::default())?; let _ = fs::remove_dir_all(root); + + assert_eq!( + provenance_line(&rendered, &index_and_artifact.runtime_key), + " provenance: channel=release source=https://example.invalid/simple \ + source_layout_generation=unrecorded device_target=unrecorded", + "the index must win over the artifact URL:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &composition_layout.runtime_key), + " provenance: channel=release source=https://example.invalid/therock \ + source_layout_generation=next-v1 device_target=unrecorded", + "a layout recorded only in the composition is still recorded:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &spec_target.runtime_key), + " provenance: channel=release source=https://example.invalid/therock \ + source_layout_generation=multi-arch-v2 device_target=gfx1201", + "a target recorded only in the package specs is still recorded:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &adopted.runtime_key), + " provenance: channel=release source=unrecorded \ + source_layout_generation=unrecorded device_target=unrecorded", + "the adoption marker is not a package source:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &first_in_chain_wins.runtime_key), + " provenance: channel=release source=https://example.invalid/therock \ + source_layout_generation=multi-arch-v2 device_target=gfx1100", + "the first recorded value in each chain must win:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &blank_before_fallback.runtime_key), + " provenance: channel=release source=unrecorded \ + source_layout_generation=unrecorded device_target=unrecorded", + "a blank first value reads unrecorded, not the field after it:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &empty_device_extra.runtime_key), + " provenance: channel=release source=https://example.invalid/therock \ + source_layout_generation=multi-arch-v2 device_target=unrecorded", + "an empty device extra names no target:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &padded.runtime_key), + " provenance: channel=release source=https://example.invalid/simple \ + source_layout_generation=multi-arch-v2 device_target=gfx1201", + "shown values must be trimmed:\n{rendered}" + ); + assert_eq!( + provenance_line(&rendered, &all_blank.runtime_key), + " provenance: channel=unrecorded source=unrecorded \ + source_layout_generation=unrecorded device_target=unrecorded", + "a fact recorded only as blank must read unrecorded:\n{rendered}" + ); + Ok(()) + } + + /// `rocm examine` reports the active runtime's provenance as one line per + /// fact. They must be the facts and words the listing shows for that same + /// runtime, so this compares the two outputs as well as pinning each. + /// Two active runtimes: a nightly install that recorded every fact, and an + /// adopted one whose manifest holds only the adoption marker where an + /// artifact URL would be, which must not be reported as its source. + #[test] + fn examine_names_the_active_runtime_provenance_the_listing_shows() -> Result<()> { + type Build = fn(therock::InstalledRuntimeManifest) -> therock::InstalledRuntimeManifest; + let adopted: Build = |base| therock::InstalledRuntimeManifest { + index_url: None, + selected_artifact_url: crate::runtime_registry::ADOPTED_RUNTIME_ARTIFACT_URL.to_owned(), + read_only: true, + imported_from: Some(base.install_root.clone()), + ..base + }; + let cases: [(&str, &str, &str, Build, [&str; 4]); 2] = [ + ( + "nightly-wheel-gfx1201-10-1-0a20260822", + "therock-nightly:gfx120X-all", + "10.1.0a20260822", + nightly_exact_target_manifest, + [ + " active_runtime_channel: nightly", + " active_runtime_source: https://example.invalid/whl-multi-arch", + " active_runtime_source_layout_generation: next-v1", + " active_runtime_device_target: gfx1201", + ], + ), + ( + "adopted-release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + adopted, + [ + " active_runtime_channel: release", + " active_runtime_source: unrecorded", + " active_runtime_source_layout_generation: unrecorded", + " active_runtime_device_target: unrecorded", + ], + ), + ]; + for (runtime_key, runtime_id, version, build, expected) in cases { + let (root, paths) = test_paths("examine-runtime-provenance"); + let base = write_test_pip_runtime(&paths, runtime_key, runtime_id, version, 10)?; + let manifest = build(base); + rewrite_manifest(&paths, &manifest)?; + let config = RocmCliConfig { + active_runtime_key: Some(manifest.runtime_key.clone()), + ..RocmCliConfig::default() + }; + + let listed = render_runtimes_text(&paths, &config)?; + let mut examined = String::new(); + append_examine_runtime_state(&mut examined, &paths, &config)?; + let _ = fs::remove_dir_all(root); + + let listed_pairs = provenance_line(&listed, &manifest.runtime_key) + .trim_start() + .strip_prefix("provenance: ") + .expect("provenance prefix") + .split(' ') + .map(|pair| pair.split_once('=').expect("name=value pair")) + .map(|(name, value)| format!(" active_runtime_{name}: {value}")) + .collect::>(); + assert_eq!(listed_pairs, expected, "{runtime_key}:\n{listed}"); + for line in expected { + assert!( + examined.lines().any(|examined_line| examined_line == line), + "{runtime_key}: examine must report {line:?} as the listing does:\n{examined}" + ); + } + } + Ok(()) + } + + #[test] + fn runtime_lists_display_build_date_from_version_string() -> Result<()> { + let (root, paths) = test_paths("runtime-build-date-display"); + let runtime_key = "release-pip-gfx120x-all-7-14-0a20260601"; + let manifest = write_test_pip_runtime( + &paths, + runtime_key, + "therock-release:gfx120X-all", + "7.14.0a20260601", + 20, + )?; + let config = RocmCliConfig { + active_runtime_key: Some(manifest.runtime_key.clone()), + default_runtime_id: Some(manifest.runtime_id), + ..RocmCliConfig::default() + }; + + let runtimes = render_runtimes_text(&paths, &config)?; + assert!(runtimes.contains("version=7.14.0a20260601 (build 2026-06-01)")); + + let mut examine = String::new(); + append_examine_runtime_state(&mut examine, &paths, &config)?; + assert!(examine.contains("active_runtime_version: 7.14.0a20260601 (build 2026-06-01)")); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// What `rocm runtimes list --json` prints for `config`, as the JSON value a + /// script reads. + fn runtimes_list_json(paths: &AppPaths, config: &RocmCliConfig) -> Result { + Ok(serde_json::to_value(runtimes_listing(paths, config)?)?) + } + + /// The `runtime_key`s whose `flag` is true, in listing order. + fn runtime_keys_flagged(listing: &serde_json::Value, flag: &str) -> Vec { + listing["runtimes"] + .as_array() + .expect("runtimes is an array") + .iter() + .filter(|runtime| { + // Every runtime must carry `flag` as a boolean: read as + // `runtime[flag] == true`, a missing or renamed field would + // count as false and "nothing is flagged" would still pass. + runtime + .get(flag) + .and_then(serde_json::Value::as_bool) + .unwrap_or_else(|| panic!("a runtime has no boolean `{flag}`:\n{listing:#}")) + }) + .map(|runtime| { + runtime["runtime_key"] + .as_str() + .unwrap_or_default() + .to_owned() + }) + .collect() + } + + fn listed_runtime<'a>( + listing: &'a serde_json::Value, + runtime_key: &str, + ) -> &'a serde_json::Value { + listing["runtimes"] + .as_array() + .expect("runtimes is an array") + .iter() + .find(|runtime| runtime["runtime_key"] == runtime_key) + .unwrap_or_else(|| panic!("{runtime_key} is listed:\n{listing:#}")) + } + + #[test] + fn runtimes_list_json_with_nothing_registered_lists_nothing_and_no_active_runtime() -> Result<()> + { + let (root, paths) = test_paths("runtime-list-json-empty"); + let listing = runtimes_list_json(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + + assert_eq!(listing["runtimes"], serde_json::json!([])); + for unset in [ + "default_runtime_id", + "active_runtime_key", + "previous_runtime_key", + "active_problem", + ] { + assert_null_field(&listing, unset); + } + for path in ["registry", "marker"] { + assert!( + listing.get(path).is_some_and(serde_json::Value::is_string), + "`{path}` is always a path:\n{listing:#}" + ); + } + assert!( + listing.get("next_step").is_none(), + "the next-step hint is text-only:\n{listing:#}" + ); + Ok(()) + } + + #[test] + fn runtimes_list_json_flags_the_active_runtime_and_the_rollback_target() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-active-rollback"); + for (key, version, build) in [ + ("rt-first", "7.12.0", 10), + ("rt-second", "7.13.0", 20), + ("rt-idle", "7.11.0", 5), + ] { + write_test_pip_runtime(&paths, key, "therock-release:gfx120X-all", version, build)?; + } + let config = RocmCliConfig { + active_runtime_key: Some("rt-second".to_owned()), + previous_runtime_key: Some("rt-first".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!(runtime_keys_flagged(&listing, "active"), ["rt-second"]); + assert_eq!( + runtime_keys_flagged(&listing, "rollback_target"), + ["rt-first"] + ); + assert_eq!(listing["active_runtime_key"], "rt-second"); + assert_eq!(listing["previous_runtime_key"], "rt-first"); + assert_null_field(&listing, "active_problem"); + let second = listed_runtime(&listing, "rt-second"); + assert_eq!(second["runtime_id"], "therock-release:gfx120X-all"); + assert_eq!(second["version"], "7.13.0"); + assert_eq!(second["format"], "wheel"); + assert_eq!(second["family"], "gfx120X-all"); + assert_eq!(second["mode"], "managed"); + assert_eq!(second["status"], "ready"); + assert_null_field(second, "unusable_reason"); + assert_eq!(second["toolchain"], "included"); + Ok(()) + } + + #[test] + fn runtimes_list_json_marks_the_one_runtime_matching_the_default_id_active() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-default-id"); + write_test_pip_runtime( + &paths, + "rt-gfx120x", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + write_test_pip_runtime( + &paths, + "rt-gfx110x", + "therock-release:gfx110X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!(runtime_keys_flagged(&listing, "active"), ["rt-gfx120x"]); + assert_eq!(listing["default_runtime_id"], "therock-release:gfx120X-all"); + assert_null_field(&listing, "active_runtime_key"); + assert_null_field(&listing, "active_problem"); + Ok(()) + } + + #[test] + fn runtimes_list_json_reports_an_active_key_with_no_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-missing-key"); + write_test_pip_runtime( + &paths, + "rt-present", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + active_runtime_key: Some("rt-gone".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + listing["active_problem"], + serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": "rt-gone"}) + ); + assert!(runtime_keys_flagged(&listing, "active").is_empty()); + Ok(()) + } + + #[test] + fn runtimes_list_json_reports_a_default_id_with_no_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-missing-id"); + write_test_pip_runtime( + &paths, + "rt-present", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx110X-all".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + listing["active_problem"], + serde_json::json!({ + "kind": "missing_manifest_for_id", + "runtime_id": "therock-release:gfx110X-all" + }) + ); + assert!(runtime_keys_flagged(&listing, "active").is_empty()); + Ok(()) + } + + #[test] + fn runtimes_list_json_reports_a_default_id_matching_two_runtimes_as_ambiguous() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-ambiguous-id"); + write_test_pip_runtime( + &paths, + "rt-newer", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + write_test_pip_runtime( + &paths, + "rt-older", + "therock-release:gfx120X-all", + "7.12.0", + 10, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let text = render_runtimes_text(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + listing["active_problem"], + serde_json::json!({ + "kind": "ambiguous_runtime_id", + "runtime_id": "therock-release:gfx120X-all", + "runtime_keys": ["rt-newer", "rt-older"] + }) + ); + assert!(runtime_keys_flagged(&listing, "active").is_empty()); + // The text form prints the same problem from the same value. + assert!( + text.contains( + " active_status: ambiguous runtime_id=therock-release:gfx120X-all; \ + activate one runtime_key: rt-newer, rt-older\n" + ), + "{text}" + ); + Ok(()) + } + + /// `key` is present in `value` and `null`. Indexing a `serde_json::Value` + /// returns `null` for a missing key too, so `value[key] == Value::Null` + /// would stay green if the key were dropped from the document. + fn assert_null_field(value: &serde_json::Value, key: &str) { + assert_eq!( + value.as_object().and_then(|object| object.get(key)), + Some(&serde_json::Value::Null), + "`{key}` must be present and null:\n{value:#}" + ); + } + + /// The `runtime_key`s the text form marks with `glyph`, in listing order. + fn runtime_keys_marked(text: &str, glyph: &str) -> Vec { + text.lines() + .filter_map(|line| line.strip_prefix(&format!(" {glyph} "))) + .filter_map(|row| row.split_whitespace().next()) + .map(str::to_owned) + .collect() + } + + /// With no `active_runtime_key`, the listing matches `default_runtime_id` + /// ignoring ASCII letter case, and `active` and `active_problem` use that + /// one rule (`default_runtime_id_matches`). A default id that differs from + /// its one runtime's id only in case marks that runtime active and reports + /// no problem; `rocm config set-default-runtime` stores the id as typed, + /// so this state is one command away. Because both fields use the same + /// rule, `active_problem` never reports as missing a runtime that `active` + /// marks. + #[test] + fn runtimes_list_matches_a_default_id_differing_only_in_case_with_one_rule() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-default-id-case"); + write_test_pip_runtime( + &paths, + "rt-gfx120x", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("THEROCK-RELEASE:GFX120X-ALL".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let text = render_runtimes_text(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!(runtime_keys_flagged(&listing, "active"), ["rt-gfx120x"]); + assert_null_field(&listing, "active_problem"); + assert_eq!( + runtime_keys_marked(&text, ACTIVE_RUNTIME_MARKER), + ["rt-gfx120x"] + ); + assert!(!text.contains("active_status:"), "{text}"); + Ok(()) + } + + /// Under the same rule, a default id matching two runtime ids that differ + /// only in letter case matches both: neither is active, and the listing + /// reports the id as ambiguous, naming both keys in listing order. The + /// runtime whose id equals the default exactly counts as one match among + /// the two, not as the winner. + #[test] + fn runtimes_list_reports_a_default_id_matching_ids_differing_only_in_case_as_ambiguous() + -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-ambiguous-id-case"); + write_test_pip_runtime( + &paths, + "rt-upper", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + write_test_pip_runtime( + &paths, + "rt-lower", + "therock-release:gfx120x-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let text = render_runtimes_text(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + listing["active_problem"], + serde_json::json!({ + "kind": "ambiguous_runtime_id", + "runtime_id": "therock-release:gfx120X-all", + "runtime_keys": ["rt-upper", "rt-lower"] + }) + ); + assert!(runtime_keys_flagged(&listing, "active").is_empty()); + assert!( + text.contains( + " active_status: ambiguous runtime_id=therock-release:gfx120X-all; \ + activate one runtime_key: rt-upper, rt-lower\n" + ), + "{text}" + ); + Ok(()) + } + + // ── State matrix: what the listing marks, for every configuration ── + // + // Built in memory through `listing_from_manifests`, so keys that differ + // only in letter case are listed as they would be on a case-sensitive + // filesystem without writing them to one: on a case-insensitive one the + // second `.json` would overwrite the first. + + /// A registered runtime with just the fields the listing reads. Its + /// install root does not exist, so it is listed `unusable`, which no row + /// below depends on. + fn listed_runtime_manifest( + runtime_key: &str, + runtime_id: &str, + ) -> therock::InstalledRuntimeManifest { + serde_json::from_value(serde_json::json!({ + "runtime_key": runtime_key, + "runtime_id": runtime_id, + "channel": "release", + "format": "wheel", + "family": "gfx120X-all", + "family_source": "test", + "version": "7.13.0", + "install_root": format!("/nonexistent/rocm-list-state/{runtime_key}"), + "selected_artifact_url": "https://example.invalid/therock", + "installed_at_unix_ms": 10, + })) + .expect("a minimal runtime manifest deserializes") + } + + const ID_A: &str = "therock-release:gfx120X-all"; + const ID_B: &str = "therock-release:gfx110X-all"; + + /// Runtimes `rt-a` (id `ID_A`) and `rt-b` (id `ID_B`), in that order. + fn two_distinct_runtimes() -> Vec { + vec![ + listed_runtime_manifest("rt-a", ID_A), + listed_runtime_manifest("rt-b", ID_B), + ] + } + + /// Keys `rt-twin` and `RT-TWIN`, in that order, with distinct ids. + fn case_twin_runtimes() -> Vec { + vec![ + listed_runtime_manifest("rt-twin", ID_A), + listed_runtime_manifest("RT-TWIN", ID_B), + ] + } + + /// One row of the state matrix: a configuration, the runtimes, and what + /// the listing must say about them in both forms. + struct ListingState { + config: RocmCliConfig, + manifests: Vec, + /// The key marked `active` (JSON) and `*` (text). + active: Option<&'static str>, + /// The key flagged `rollback_target` (JSON); marked `-` in the text + /// unless it is also active. + rollback: Option<&'static str>, + /// The JSON `active_problem`. + problem: serde_json::Value, + /// The text's `active_status:` line, without its indentation. + status_line: Option<&'static str>, + } + + fn config_with( + default_runtime_id: Option<&str>, + active_runtime_key: Option<&str>, + previous_runtime_key: Option<&str>, + ) -> RocmCliConfig { + RocmCliConfig { + default_runtime_id: default_runtime_id.map(str::to_owned), + active_runtime_key: active_runtime_key.map(str::to_owned), + previous_runtime_key: previous_runtime_key.map(str::to_owned), + ..RocmCliConfig::default() + } + } + + fn assert_listing_state(state: &ListingState) -> Result<()> { + use cli_report::JsonOrText as _; + let listing = listing_from_manifests( + &state.config, + state.manifests.clone(), + PathBuf::from("registry"), + PathBuf::from("active.json"), + ); + let json = serde_json::to_value(&listing)?; + let text = listing.render_text(); + let context = format!("{json:#}\n{text}"); + + // The listing marks what the commands resolve: `current_runtime_manifest` + // for the active runtime, `select_runtime_manifest` for rollback. + let resolved_active = current_runtime_manifest(&state.config, &state.manifests) + .map(|manifest| manifest.runtime_key.as_str()); + assert_eq!(resolved_active, state.active, "resolver vs row:\n{context}"); + let resolved_rollback = state + .config + .previous_runtime_key + .as_deref() + .and_then(|selector| select_runtime_manifest(&state.manifests, selector).ok()) + .map(|manifest| manifest.runtime_key.as_str()); + assert_eq!( + resolved_rollback, state.rollback, + "resolver vs row:\n{context}" + ); + + let expected_active: Vec = state.active.iter().map(|k| (*k).to_owned()).collect(); + let expected_rollback: Vec = + state.rollback.iter().map(|k| (*k).to_owned()).collect(); + assert_eq!( + runtime_keys_flagged(&json, "active"), + expected_active, + "{context}" + ); + assert_eq!( + runtime_keys_flagged(&json, "rollback_target"), + expected_rollback, + "{context}" + ); + assert_eq!(json["active_problem"], state.problem, "{context}"); + assert_eq!( + runtime_keys_marked(&text, ACTIVE_RUNTIME_MARKER), + expected_active, + "{context}" + ); + let rollback_only: Vec = expected_rollback + .into_iter() + .filter(|key| Some(key.as_str()) != state.active) + .collect(); + assert_eq!( + runtime_keys_marked(&text, ROLLBACK_RUNTIME_MARKER), + rollback_only, + "{context}" + ); + let status_lines: Vec<&str> = text + .lines() + .map(str::trim_start) + .filter(|line| line.starts_with("active_status:")) + .collect(); + assert_eq!( + status_lines, + state.status_line.into_iter().collect::>(), + "{context}" + ); + Ok(()) + } + + fn state_no_config() -> ListingState { + ListingState { + config: RocmCliConfig::default(), + manifests: two_distinct_runtimes(), + active: None, + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_key_exact() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), None), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// The default id names the other runtime, so the key alone decides. + fn state_key_in_other_case() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("RT-A"), None), + manifests: two_distinct_runtimes(), + active: Some("rt-a"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// The one documented case where `active_problem` is set beside an + /// `active` runtime: the key is dead, the default id still resolves. + fn state_stale_key_default_resolves() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-gone"), None), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: None, + problem: serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": "rt-gone"}), + status_line: Some("active_status: missing manifest for active_runtime_key=rt-gone"), + } + } + + fn state_stale_key_default_missing() -> ListingState { + ListingState { + config: config_with(Some("therock-release:gfx90a"), Some("rt-gone"), None), + manifests: two_distinct_runtimes(), + active: None, + rollback: None, + problem: serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": "rt-gone"}), + status_line: Some("active_status: missing manifest for active_runtime_key=rt-gone"), + } + } + + fn state_stale_key_default_ambiguous() -> ListingState { + ListingState { + config: config_with(Some(ID_A), Some("rt-gone"), None), + manifests: vec![ + listed_runtime_manifest("rt-a", ID_A), + listed_runtime_manifest("rt-a2", ID_A), + ], + active: None, + rollback: None, + problem: serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": "rt-gone"}), + status_line: Some("active_status: missing manifest for active_runtime_key=rt-gone"), + } + } + + fn state_default_matches_none() -> ListingState { + ListingState { + config: config_with(Some("therock-release:gfx90a"), None, None), + manifests: two_distinct_runtimes(), + active: None, + rollback: None, + problem: serde_json::json!({ + "kind": "missing_manifest_for_id", + "runtime_id": "therock-release:gfx90a" + }), + status_line: Some( + "active_status: missing manifest for active_runtime_id=therock-release:gfx90a", + ), + } + } + + fn state_default_matches_one() -> ListingState { + ListingState { + config: config_with(Some(ID_B), None, None), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_default_matches_one_in_other_case() -> ListingState { + ListingState { + config: config_with(Some("THEROCK-RELEASE:GFX110X-ALL"), None, None), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_default_matches_two() -> ListingState { + ListingState { + config: config_with(Some(ID_A), None, None), + manifests: vec![ + listed_runtime_manifest("rt-a", ID_A), + listed_runtime_manifest("rt-a2", ID_A), + ], + active: None, + rollback: None, + problem: serde_json::json!({ + "kind": "ambiguous_runtime_id", + "runtime_id": ID_A, + "runtime_keys": ["rt-a", "rt-a2"] + }), + status_line: Some( + "active_status: ambiguous runtime_id=therock-release:gfx120X-all; \ + activate one runtime_key: rt-a, rt-a2", + ), + } + } + + /// Ids differing only in case are two matches, even though one equals the + /// default exactly. + fn state_default_matches_two_differing_in_case() -> ListingState { + ListingState { + config: config_with(Some(ID_A), None, None), + manifests: vec![ + listed_runtime_manifest("rt-a", ID_A), + listed_runtime_manifest("rt-a-lower", "therock-release:gfx120x-all"), + ], + active: None, + rollback: None, + problem: serde_json::json!({ + "kind": "ambiguous_runtime_id", + "runtime_id": ID_A, + "runtime_keys": ["rt-a", "rt-a-lower"] + }), + status_line: Some( + "active_status: ambiguous runtime_id=therock-release:gfx120X-all; \ + activate one runtime_key: rt-a, rt-a-lower", + ), + } + } + + /// Case twins: the resolver takes the first key matching in list order, + /// even when a later one equals the active key exactly, and the listing + /// marks that first one, because that is what the commands act on. + fn state_case_twins_key_names_the_second() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("RT-TWIN"), None), + manifests: case_twin_runtimes(), + active: Some("rt-twin"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_case_twins_key_names_the_first() -> ListingState { + ListingState { + config: config_with(Some(ID_A), Some("rt-twin"), None), + manifests: case_twin_runtimes(), + active: Some("rt-twin"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_previous_key_exact() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), Some("rt-a")), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: Some("rt-a"), + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_previous_key_in_other_case() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), Some("RT-A")), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: Some("rt-a"), + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_previous_key_is_a_runtime_id() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), Some(ID_A)), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: Some("rt-a"), + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_previous_key_with_whitespace() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), Some(" rt-a\n")), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: Some("rt-a"), + problem: serde_json::Value::Null, + status_line: None, + } + } + + fn state_previous_key_missing() -> ListingState { + ListingState { + config: config_with(Some(ID_B), Some("rt-b"), Some("rt-gone")), + manifests: two_distinct_runtimes(), + active: Some("rt-b"), + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// Matches both twins only by letter case: rollback refuses it as + /// ambiguous, so nothing is the rollback target. + fn state_previous_key_ambiguous_case_twins() -> ListingState { + ListingState { + config: config_with(None, None, Some("Rt-Twin")), + manifests: case_twin_runtimes(), + active: None, + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// A runtime id two runtimes share: rollback refuses it as ambiguous. + fn state_previous_key_ambiguous_runtime_id() -> ListingState { + ListingState { + config: config_with(None, None, Some(ID_A)), + manifests: vec![ + listed_runtime_manifest("rt-a", ID_A), + listed_runtime_manifest("rt-a2", ID_A), + ], + active: None, + rollback: None, + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// Rollback takes an exact key over its case twin. + fn state_previous_key_names_one_case_twin_exactly() -> ListingState { + ListingState { + config: config_with(None, None, Some("RT-TWIN")), + manifests: case_twin_runtimes(), + active: None, + rollback: Some("RT-TWIN"), + problem: serde_json::Value::Null, + status_line: None, + } + } + + /// The active runtime's registry entry was deleted and the default id it + /// shared with the rollback target now resolves that one: it is both, and + /// the text marks it `*`. + fn state_stale_key_falls_back_to_the_rollback_target() -> ListingState { + ListingState { + config: config_with(Some(ID_A), Some("rt-gone"), Some("rt-a")), + manifests: two_distinct_runtimes(), + active: Some("rt-a"), + rollback: Some("rt-a"), + problem: serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": "rt-gone"}), + status_line: Some("active_status: missing manifest for active_runtime_key=rt-gone"), + } + } + + #[test] + fn listing_state_no_config() -> Result<()> { + assert_listing_state(&state_no_config()) + } + + #[test] + fn listing_state_key_exact() -> Result<()> { + assert_listing_state(&state_key_exact()) + } + + #[test] + fn listing_state_key_in_other_case() -> Result<()> { + assert_listing_state(&state_key_in_other_case()) + } + + #[test] + fn listing_state_stale_key_default_resolves() -> Result<()> { + assert_listing_state(&state_stale_key_default_resolves()) + } + + #[test] + fn listing_state_stale_key_default_missing() -> Result<()> { + assert_listing_state(&state_stale_key_default_missing()) + } + + #[test] + fn listing_state_stale_key_default_ambiguous() -> Result<()> { + assert_listing_state(&state_stale_key_default_ambiguous()) + } + + #[test] + fn listing_state_default_matches_none() -> Result<()> { + assert_listing_state(&state_default_matches_none()) + } + + #[test] + fn listing_state_default_matches_one() -> Result<()> { + assert_listing_state(&state_default_matches_one()) + } + + #[test] + fn listing_state_default_matches_one_in_other_case() -> Result<()> { + assert_listing_state(&state_default_matches_one_in_other_case()) + } + + #[test] + fn listing_state_default_matches_two() -> Result<()> { + assert_listing_state(&state_default_matches_two()) + } + + #[test] + fn listing_state_default_matches_two_differing_in_case() -> Result<()> { + assert_listing_state(&state_default_matches_two_differing_in_case()) + } + + #[test] + fn listing_state_case_twins_key_names_the_second() -> Result<()> { + assert_listing_state(&state_case_twins_key_names_the_second()) + } + + #[test] + fn listing_state_case_twins_key_names_the_first() -> Result<()> { + assert_listing_state(&state_case_twins_key_names_the_first()) + } + + #[test] + fn listing_state_previous_key_exact() -> Result<()> { + assert_listing_state(&state_previous_key_exact()) + } + + #[test] + fn listing_state_previous_key_in_other_case() -> Result<()> { + assert_listing_state(&state_previous_key_in_other_case()) + } + + #[test] + fn listing_state_previous_key_is_a_runtime_id() -> Result<()> { + assert_listing_state(&state_previous_key_is_a_runtime_id()) + } + + #[test] + fn listing_state_previous_key_with_whitespace() -> Result<()> { + assert_listing_state(&state_previous_key_with_whitespace()) + } + + #[test] + fn listing_state_previous_key_missing() -> Result<()> { + assert_listing_state(&state_previous_key_missing()) + } + + #[test] + fn listing_state_previous_key_ambiguous_case_twins() -> Result<()> { + assert_listing_state(&state_previous_key_ambiguous_case_twins()) + } + + #[test] + fn listing_state_previous_key_ambiguous_runtime_id() -> Result<()> { + assert_listing_state(&state_previous_key_ambiguous_runtime_id()) + } + + #[test] + fn listing_state_previous_key_names_one_case_twin_exactly() -> Result<()> { + assert_listing_state(&state_previous_key_names_one_case_twin_exactly()) + } + + #[test] + fn listing_state_stale_key_falls_back_to_the_rollback_target() -> Result<()> { + assert_listing_state(&state_stale_key_falls_back_to_the_rollback_target()) + } + + /// A data dir whose path is not valid UTF-8 still lists as JSON: the + /// paths are written lossily, as the text form writes them, instead of + /// failing serialization and leaving stdout empty. + #[cfg(unix)] + #[test] + fn runtimes_list_json_writes_a_non_utf8_path_lossily() -> Result<()> { + use std::os::unix::ffi::OsStrExt as _; + let data_dir = PathBuf::from(std::ffi::OsStr::from_bytes(b"/data-\xff")); + let mut runtime = listed_runtime_manifest("rt-a", ID_A); + runtime.install_root = data_dir.join("rt-a"); + let listing = listing_from_manifests( + &RocmCliConfig::default(), + vec![runtime], + data_dir.join("registry"), + data_dir.join("active.json"), + ); + let json = serde_json::to_value(&listing)?; + assert_eq!(json["registry"], "/data-\u{fffd}/registry", "{json:#}"); + assert_eq!(json["marker"], "/data-\u{fffd}/active.json", "{json:#}"); + assert_eq!( + listed_runtime(&json, "rt-a")["install_root"], + "/data-\u{fffd}/rt-a", + "{json:#}" + ); + Ok(()) + } + + /// The text prints `toolchain=` from `RuntimeToolchain::as_str`, which is + /// `toolchain_state_text`, the word `rocm examine` prints; the JSON prints + /// the serde `lowercase` rename. Both are documented as `included` and + /// `excluded`, so a change to either word fails here instead of silently + /// changing the JSON. As for the mode test below, a new variant must be + /// added to `all` as well as to the match. + #[test] + fn runtime_toolchain_text_and_json_words_agree_for_every_variant() -> Result<()> { + let all = [RuntimeToolchain::Included, RuntimeToolchain::Excluded]; + for toolchain in all { + match toolchain { + RuntimeToolchain::Included | RuntimeToolchain::Excluded => {} + } + assert_eq!( + serde_json::to_value(toolchain)?, + toolchain.as_str(), + "{toolchain:?}" + ); + } + assert_eq!( + RuntimeToolchain::from_includes_devel(true).as_str(), + "included" + ); + assert_eq!( + RuntimeToolchain::from_includes_devel(false).as_str(), + "excluded" + ); + Ok(()) + } + + /// The text prints `mode=` from `RuntimeMode::as_str` and the JSON from the + /// serde `kebab-case` rename; both are documented as the same words, so + /// every variant must agree. The exhaustive `match` turns a new variant + /// into a compile error here, which forces a look at this test, but it + /// does not add the variant to `all`: whoever adds the match arm must add + /// the variant to `all` too. + #[test] + fn runtime_mode_text_and_json_words_agree_for_every_variant() -> Result<()> { + let all = [RuntimeMode::Managed, RuntimeMode::ReadOnly]; + for mode in all { + match mode { + RuntimeMode::Managed | RuntimeMode::ReadOnly => {} + } + assert_eq!(serde_json::to_value(mode)?, mode.as_str(), "{mode:?}"); + } + Ok(()) + } + + /// The documented shape of `rocm runtimes list --json` (docs/json-output.md): + /// exactly these top-level and per-runtime keys, the registry and marker + /// paths, each runtime's own install root, and newest install first with + /// ties broken by `runtime_key`. + #[test] + fn runtimes_list_json_has_exactly_the_documented_fields_newest_install_first() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-contract"); + let mut install_roots = std::collections::BTreeMap::new(); + // Five runtimes share one install time and are planted in reverse key + // order, so the order the registry is read in cannot pass for the + // `runtime_key` tie-break by accident. + for (key, build) in [ + ("rt-old", 10), + ("rt-tie-e", 20), + ("rt-tie-d", 20), + ("rt-new", 30), + ("rt-tie-c", 20), + ("rt-tie-b", 20), + ("rt-tie-a", 20), + ] { + let manifest = write_test_pip_runtime( + &paths, + key, + "therock-release:gfx120X-all", + "7.13.0", + build, + )?; + install_roots.insert(key, manifest.install_root); + } + let listing = runtimes_list_json(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(&root); + + // Compared as sets: the key order of a `serde_json::Map` depends on + // whether its `preserve_order` feature is on, and is not documented. + let key_set = |value: &serde_json::Value| -> std::collections::BTreeSet { + value + .as_object() + .expect("an object") + .keys() + .cloned() + .collect() + }; + assert_eq!( + key_set(&listing), + std::collections::BTreeSet::from( + [ + "default_runtime_id", + "active_runtime_key", + "previous_runtime_key", + "registry", + "marker", + "active_problem", + "runtimes", + ] + .map(String::from) + ) + ); + assert_eq!( + listing["registry"], + serde_json::to_value(runtime_registry_dir(&paths))? + ); + assert_eq!( + listing["marker"], + serde_json::to_value(active_runtime_marker_path(&paths))? + ); + + let runtimes = listing["runtimes"] + .as_array() + .expect("runtimes is an array"); + let order: Vec<&str> = runtimes + .iter() + .map(|runtime| runtime["runtime_key"].as_str().unwrap_or_default()) + .collect(); + assert_eq!( + order, + [ + "rt-new", "rt-tie-a", "rt-tie-b", "rt-tie-c", "rt-tie-d", "rt-tie-e", "rt-old", + ] + ); + for runtime in runtimes { + assert_eq!( + key_set(runtime), + std::collections::BTreeSet::from( + [ + "runtime_key", + "runtime_id", + "version", + "format", + "family", + "mode", + "status", + "unusable_reason", + "toolchain", + "active", + "rollback_target", + "install_root", + "provenance", + ] + .map(String::from) + ) + ); + assert_eq!( + key_set(&runtime["provenance"]), + std::collections::BTreeSet::from( + [ + "channel", + "source", + "source_layout_generation", + "device_target", + ] + .map(String::from) + ) + ); + let key = runtime["runtime_key"].as_str().unwrap_or_default(); + assert_eq!( + runtime["install_root"], + serde_json::to_value(&install_roots[key])?, + "{key}" + ); + } + Ok(()) + } + + /// Plants the runtimes whose provenance the JSON tests read: one per way a + /// fact can be recorded or not. Returns their keys, in this order: a + /// nightly that recorded every fact, a runtime from before the layout and + /// target were recorded, an adopted one (the adoption marker is not a + /// source), a tarball install (no exact target), one whose values are + /// padded with whitespace, and one whose every value is blank. + fn plant_provenance_runtimes(paths: &AppPaths) -> Result<[String; 6]> { + let base = write_test_pip_runtime( + paths, + "release-pip-legacy", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let legacy = therock::InstalledRuntimeManifest { + index_url: None, + source_layout_generation: None, + wheel_composition: None, + ..base.clone() + }; + let nightly = nightly_exact_target_manifest(base); + write_test_pip_runtime( + paths, + &nightly.runtime_key, + &nightly.runtime_id, + &nightly.version, + 20, + )?; + let adopted = therock::InstalledRuntimeManifest { + runtime_key: "adopted-release-pip".to_owned(), + index_url: None, + selected_artifact_url: crate::runtime_registry::ADOPTED_RUNTIME_ARTIFACT_URL.to_owned(), + read_only: true, + ..legacy.clone() + }; + let tarball = therock::InstalledRuntimeManifest { + runtime_key: "release-tarball-gfx120x-all-7-13-0".to_owned(), + format: "tarball".to_owned(), + selected_artifact_url: "https://example.invalid/tarball/rocm-7.13.0.tar.gz".to_owned(), + source_layout_generation: Some("multi-arch-v2".to_owned()), + ..legacy.clone() + }; + let padded = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-padded".to_owned(), + channel: " release ".to_owned(), + index_url: Some(" https://example.invalid/simple ".to_owned()), + source_layout_generation: Some(" multi-arch-v2 ".to_owned()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: "multi-arch-v2".to_owned(), + package_specs: vec!["rocm[libraries]==7.13.0".to_owned()], + rocm_sdk_target: Some(" gfx1201 ".to_owned()), + }), + ..legacy.clone() + }; + let all_blank = therock::InstalledRuntimeManifest { + runtime_key: "release-pip-all-blank".to_owned(), + channel: String::new(), + index_url: Some(" ".to_owned()), + selected_artifact_url: String::new(), + source_layout_generation: Some(String::new()), + wheel_composition: Some(therock::WheelRuntimeComposition { + source_layout_generation: String::new(), + package_specs: vec!["rocm[libraries]==7.13.0".to_owned()], + rocm_sdk_target: Some(String::new()), + }), + ..legacy.clone() + }; + let planted = [nightly, legacy, adopted, tarball, padded, all_blank]; + for manifest in &planted { + rewrite_manifest(paths, manifest)?; + } + Ok(planted.map(|manifest| manifest.runtime_key)) + } + + /// The JSON names each runtime's provenance facts as strings, trimmed, and + /// writes `null` for one recorded nowhere or recorded blank — never the + /// text's `unrecorded` placeholder, which a script would read as a value. + /// Each object is compared whole, so a fact under the wrong name, a missing + /// one or a placeholder in place of `null` fails here. + #[test] + fn runtimes_list_json_names_recorded_provenance_and_null_for_unrecorded() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-provenance"); + let [nightly, legacy, adopted, tarball, padded, all_blank] = + plant_provenance_runtimes(&paths)?; + let listing = runtimes_list_json(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + + let provenance = |key: &str| listed_runtime(&listing, key)["provenance"].clone(); + for (key, expected) in [ + ( + nightly, + serde_json::json!({ + "channel": "nightly", + "source": "https://example.invalid/whl-multi-arch", + "source_layout_generation": "next-v1", + "device_target": "gfx1201", + }), + ), + ( + legacy, + serde_json::json!({ + "channel": "release", + "source": "https://example.invalid/therock", + "source_layout_generation": null, + "device_target": null, + }), + ), + ( + adopted, + serde_json::json!({ + "channel": "release", + "source": null, + "source_layout_generation": null, + "device_target": null, + }), + ), + ( + tarball, + serde_json::json!({ + "channel": "release", + "source": "https://example.invalid/tarball/rocm-7.13.0.tar.gz", + "source_layout_generation": "multi-arch-v2", + "device_target": null, + }), + ), + ( + padded, + serde_json::json!({ + "channel": "release", + "source": "https://example.invalid/simple", + "source_layout_generation": "multi-arch-v2", + "device_target": "gfx1201", + }), + ), + ( + all_blank, + serde_json::json!({ + "channel": null, + "source": null, + "source_layout_generation": null, + "device_target": null, + }), + ), + ] { + assert_eq!(provenance(&key), expected, "{key}:\n{listing:#}"); + } + Ok(()) + } + + /// The text's `provenance:` line and the JSON's `provenance` object name + /// the same facts with the same values for every runtime, the text printing + /// `unrecorded` exactly where the JSON has `null`. Both forms are rendered + /// from one listing of the same registry and compared fact by fact, over + /// every way a fact can be recorded or not. + #[test] + fn runtimes_list_text_and_json_name_the_same_provenance() -> Result<()> { + use cli_report::JsonOrText as _; + let (root, paths) = test_paths("runtime-list-provenance-agree"); + let keys = plant_provenance_runtimes(&paths)?; + let listing = runtimes_listing(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + let text = listing.render_text(); + let json = serde_json::to_value(&listing)?; + + for key in keys { + let text_pairs: Vec<(String, String)> = provenance_line(&text, &key) + .trim_start() + .strip_prefix("provenance: ") + .expect("provenance prefix") + .split(' ') + .map(|pair| pair.split_once('=').expect("name=value pair")) + .map(|(name, value)| (name.to_owned(), value.to_owned())) + .collect(); + let object = listed_runtime(&json, &key)["provenance"] + .as_object() + .unwrap_or_else(|| panic!("{key} has no provenance object:\n{json:#}")) + .clone(); + assert_eq!( + text_pairs.len(), + object.len(), + "{key}: the text and the JSON name different facts:\n{text}\n{json:#}" + ); + for (name, shown) in text_pairs { + let in_json = object + .get(&name) + .unwrap_or_else(|| panic!("{key}: the JSON has no {name}:\n{json:#}")); + let expected = match in_json { + serde_json::Value::Null => crate::runtime_registry::UNRECORDED_PROVENANCE, + serde_json::Value::String(value) => value.as_str(), + other => panic!("{key}: {name} is neither a string nor null: {other}"), + }; + assert_eq!(shown, expected, "{key}: {name}:\n{text}\n{json:#}"); + } + } + Ok(()) + } + + /// A runtime can be both the active one and the recorded rollback target + /// when the config says so (`rocm runtimes activate` and `rollback` never + /// write that, but the config file is read as it is). Both JSON flags are true; + /// the text has one marker column, and active wins it. + #[test] + fn runtimes_list_marks_a_runtime_both_active_and_rollback_target_as_active() -> Result<()> { + let (root, paths) = test_paths("runtime-list-active-and-rollback"); + write_test_pip_runtime( + &paths, + "rt-both", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig { + active_runtime_key: Some("rt-both".to_owned()), + previous_runtime_key: Some("rt-both".to_owned()), + ..RocmCliConfig::default() + }; + let listing = runtimes_list_json(&paths, &config)?; + let text = render_runtimes_text(&paths, &config)?; + let _ = fs::remove_dir_all(root); + + assert_eq!(runtime_keys_flagged(&listing, "active"), ["rt-both"]); + assert_eq!( + runtime_keys_flagged(&listing, "rollback_target"), + ["rt-both"] + ); + assert_eq!( + runtime_keys_marked(&text, ACTIVE_RUNTIME_MARKER), + ["rt-both"] + ); + assert!( + runtime_keys_marked(&text, ROLLBACK_RUNTIME_MARKER).is_empty(), + "{text}" + ); + Ok(()) + } + + #[test] + fn runtimes_list_json_reports_a_runtime_whose_install_root_is_gone_as_unusable() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-unusable"); + let manifest = write_test_pip_runtime( + &paths, + "rt-gone", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + fs::remove_dir_all(&manifest.install_root)?; + let listing = runtimes_list_json(&paths, &RocmCliConfig::default())?; + let text = render_runtimes_text(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + + let runtime = listed_runtime(&listing, "rt-gone"); + assert_eq!(runtime["status"], "unusable"); + let reason = runtime["unusable_reason"] + .as_str() + .expect("an unusable runtime says why"); + assert!(reason.starts_with("install root is missing: "), "{reason}"); + assert!( + text.contains(&format!("status=unusable ({reason}) toolchain=")), + "the text shows the same reason:\n{text}" + ); + Ok(()) + } + + #[test] + fn runtimes_list_json_reports_an_imported_runtime_as_read_only() -> Result<()> { + let (root, paths) = test_paths("runtime-list-json-read-only"); + let manifest = write_test_pip_runtime( + &paths, + "rt-imported", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let imported = therock::InstalledRuntimeManifest { + read_only: true, + ..manifest + }; + fs::write( + runtime_manifest_path(&paths, &imported.runtime_key), + serde_json::to_vec_pretty(&imported)?, + )?; + let listing = runtimes_list_json(&paths, &RocmCliConfig::default())?; + let _ = fs::remove_dir_all(root); + + assert_eq!(listed_runtime(&listing, "rt-imported")["mode"], "read-only"); Ok(()) } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2336899be7..f359699db3 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -864,8 +864,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // (probe returns `None`). The E2E-only // backend-failure scenario bypasses this host precondition so the black-box // test reaches Lemonade's backend boundary without real GPU hardware, and the - // E2E-only OOM-launch fault injection likewise stands in for the GPU it does - // not have. + // E2E-only launch fault injection (either mode) likewise stands in for the + // GPU it does not have. // // The scripted free-space override opens the same waiver: it exists to refuse a // Lemonade install on a drive that cannot hold it, which `rocm serve` only @@ -1124,7 +1124,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { let managed_runtime_id = resolved_selection.runtime_id.clone(); let managed_env_id = resolved_selection.env_id.clone(); - // Persist the endpoint key (public bind only) in a 0600 file so the engine + // Persist the endpoint key (public bind only) in an owner-only file so the engine // child, the restart/recovery path, and inspection commands can retrieve it by // service id. Loopback binds resolve to `None` and store nothing. if let Some(key) = endpoint_auth.as_deref() { @@ -1193,20 +1193,23 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }; let mut spinner = spinner.into_inner(); // Not when the launch was fabricated. `simulate_oom_managed_launch` - // writes a `starting` record for a process that never existed, so the - // helper would be left watching a pid that will never appear, with - // nothing in the scenario's teardown to stop it: observed as `rocm - // daemon` processes outliving their e2e checkout by days, still - // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for + // writes a record (`starting`, or `ready` in its `ready` mode) for a + // process that never existed, so the helper would be left watching a pid + // that will never appear, with nothing in the scenario's teardown to stop + // it: observed as `rocm daemon` processes outliving their e2e checkout by + // days, still carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for // supervising a real spawned server, and there is none here. // // That — not ordering — is the whole justification. This suppression // runs *before* the OOM note is computed in the `summary_mode` block // below, so the previous claim here, that the note "is computed before // it would have run", was backwards and could not have excused it. - // serve-40's `no background automation daemon was left behind` is what - // holds the branch: deleting it left the scenario green with a `rocm - // daemon` still running. + // The `no background automation daemon was left behind` step of the + // `@id:serve-oom-launch-memory-guidance` and + // `@id:serve-smoke-failure-summary` scenarios records that none was + // started but does not discriminate this branch: the helper starts only + // when a watcher is enabled, and the isolated config those scenarios use + // enables none, so removing the branch leaves them green. // // `e2e_simulate_oom_launch` is a `const fn` returning `false` without // the test-only feature, so this branch folds away in shipped builds. @@ -1236,16 +1239,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Best-effort inference smoke test, on by default (opt out with // `--no-smoke-test`). Only meaningful for a freshly-ready server we // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { - spinner.set_label("Running smoke test…"); - // The local provider resolves the endpoint key from the per-service - // 0600 key file by service id, so the smoke test authenticates - // against a protected public endpoint without threading the secret - // through here. - serve_summary::run_smoke_test(&paths, resolve.served_model_name()) - } else { - serve_summary::SmokeMetrics::default() - }; + let (metrics, smoke_failure) = + if !no_smoke_test && !report.already_running && report.status == "ready" { + spinner.set_label("Running smoke test…"); + // The local provider resolves the endpoint key from the per-service + // owner-only key file by service id, so the smoke test authenticates + // against a protected public endpoint without threading the secret + // through here. + serve_summary::run_smoke_test(&paths, resolve.served_model_name()).into_parts() + } else { + (serve_summary::SmokeMetrics::default(), None) + }; spinner.clear(); let mut notes = collect_serve_notes( @@ -1314,6 +1318,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { status: report.status.clone(), already_running: report.already_running, metrics, + smoke_failure, api_key: launched_key, notes, }; diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index cd0ab34bc6..c30bcc654a 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -32,7 +32,9 @@ const SMOKE_PROMPT: &str = "Reply with a short one-sentence greeting."; const SMOKE_MAX_TOKENS: u32 = 32; /// Best-effort metrics measured against a freshly-started server. Every field is -/// optional: any probe failure leaves it `None` and the summary renders `n/a`. +/// optional: `None` renders as `n/a`. They are empty when the smoke test was +/// skipped, failed (see [`SmokeOutcome::Failed`]), or completed without a content +/// token (a reasoning model can spend its whole token budget before emitting any). #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SmokeMetrics { /// Time from request send to the first streamed content token. @@ -62,6 +64,11 @@ pub(crate) struct DeploymentSummary { /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, pub metrics: SmokeMetrics, + /// Why the post-launch smoke test failed, as one sanitised line. `None` when + /// the test passed or was not run. When `Some` (and `already_running` is + /// false, which the renderer checks) the summary is not presented as ready; + /// an already-running service never ran a smoke test and ignores it. + pub smoke_failure: Option, /// The endpoint API key for a freshly launched *public* server, shown once as /// secure client configuration. `None` for loopback binds (no auth) and for an /// already-running service (whose key was delivered at its own launch). @@ -70,6 +77,70 @@ pub(crate) struct DeploymentSummary { pub notes: Vec, } +/// Result of the post-launch smoke test. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum SmokeOutcome { + /// The request completed with no transport error, no HTTP error status and + /// no truncated stream; the metrics are what was measured. They are empty + /// when no content token arrived, which is not a failure: a reasoning model + /// can spend its small token budget before emitting any, as + /// `openai_chat_completion_probe` also allows for readiness. The status check + /// is deliberately stricter than that probe's (any non-200 fails here, where + /// the probe accepts anything below 500). Not detected here, and so + /// `Measured`: an error reported inside a 200 stream as a `data:` payload, a + /// 200 answer with no `data:` line at all (HTML, plain JSON, an empty body), + /// and a close-delimited stream cut after a token. + Measured(SmokeMetrics), + /// The smoke test could not complete: a transport error, an HTTP error + /// status, a parse error, a chunked or `Content-Length` stream cut short, or + /// a failure in the provider plumbing around the request (finding the ready + /// service, writing the audit event after the stream). Carries a one-line + /// reason. It does not by itself prove the engine failed. + Failed(String), +} + +impl SmokeOutcome { + /// Splits the outcome into the summary's `metrics` and `smoke_failure` fields. + pub(crate) fn into_parts(self) -> (SmokeMetrics, Option) { + match self { + Self::Measured(metrics) => (metrics, None), + Self::Failed(reason) => (SmokeMetrics::default(), Some(reason)), + } + } +} + +/// Longest failure reason kept for display. +const SMOKE_FAILURE_MAX_CHARS: usize = 200; + +/// Reduces an error message to one printable line. On the smoke-test path the +/// engine-controlled text in it is the HTTP status line, reason phrase included +/// (the local provider reports `local provider returned {status_line}`, not the +/// response body), and the chunk-size line it quotes when a chunked body is +/// malformed (`invalid local provider chunk size`), so it is untrusted. Terminal +/// escape sequences and any control or Unicode format character (a bidi override, +/// a zero-width mark) are dropped, line breaks and runs of whitespace become a +/// single space, backticks become apostrophes (the note quotes its own command in +/// backticks, and the reason must not add a second one), and the result is +/// length-capped. +fn one_line_reason(raw: &str) -> String { + // The escape scanner, not this function, decides where a line break falls: + // turning breaks into spaces first would let a cut-off escape sequence take + // the next real character as an intermediate byte and swallow it. + let collapsed = rocm_core::terminal::rendered_lines(raw) + .join(" ") + .split_whitespace() + .collect::>() + .join(" ") + .replace('`', "'"); + if collapsed.chars().count() > SMOKE_FAILURE_MAX_CHARS { + let mut cut: String = collapsed.chars().take(SMOKE_FAILURE_MAX_CHARS).collect(); + cut.push_str("..."); + cut + } else { + collapsed + } +} + /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not /// produce a first token. pub(crate) fn format_ttft(ttft: Option) -> String { @@ -102,8 +173,11 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { // note must make that visibly different from a healthy deployment so the // summary is never mistaken for success. let ready = summary.status == "ready"; + let smoke_failed = summary.smoke_failure.is_some() && !summary.already_running; let heading = if summary.already_running { "Deployment summary (already running)" + } else if smoke_failed { + "Deployment summary (smoke test failed)" } else if ready { "Deployment summary" } else { @@ -132,7 +206,14 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { .collect(); let rows: Vec<(&str, String)> = [ vec![ - ("status", summary.status.clone()), + ( + "status", + if smoke_failed { + "started, smoke test failed".to_owned() + } else { + summary.status.clone() + }, + ), ("engine", summary.engine.clone()), ], model_rows, @@ -189,6 +270,16 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { summary.service_id ); } + if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { + // The server answered its health check, but the smoke test could not + // complete, so it is not known to serve. Say why the metrics rows are + // empty, hedged: the reason may be rocm's own plumbing, not the engine. + let _ = writeln!( + out, + " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`", + id = summary.service_id + ); + } for note in &summary.notes { let _ = writeln!(out, " note: {note}"); } @@ -280,10 +371,18 @@ pub(crate) fn oom_memory_note( } /// Run a single small chat completion against the just-started local server and -/// measure time-to-first-token and generation throughput. Best-effort: any error -/// (server not OpenAI-compatible, refused, timed out) yields empty metrics rather -/// than failing the launch. -pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics { +/// measure time-to-first-token and generation throughput. Best-effort: a request +/// that errors (refused, an HTTP error status other than 200, timed out) or +/// whose stream is a chunked or `Content-Length` body cut short yields +/// [`SmokeOutcome::Failed`] with the reason, rather than failing the launch. So +/// does any other error from the provider call, including two that are not the +/// engine's fault: no ready service found when it re-lists `/v1/models`, and the +/// audit-log write after the stream. A request that completes without a content +/// token is `Measured` with empty metrics, not a failure. Also `Measured`, and so +/// not detected: an error `data:` payload inside a 200 stream, a 200 answer with +/// no `data:` line at all, and a close-delimited stream cut after a token (it has +/// no framing to cut short). +pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome { let request = ChatRequest { model: Some(api_model.to_owned()), messages: vec![ChatMessage { @@ -316,11 +415,16 @@ pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics }, ); - if result.is_err() { - return SmokeMetrics::default(); + if let Err(err) = result { + return SmokeOutcome::Failed(one_line_reason(&format!("{err:#}"))); } - compute_metrics(start, first_token, last_token, token_chunks) + SmokeOutcome::Measured(compute_metrics( + start, + first_token, + last_token, + token_chunks, + )) } /// Pure metric math, split out so it can be unit-tested without a live server. @@ -466,6 +570,7 @@ mod tests { ttft: Some(Duration::from_millis(180)), gen_tps: Some(42.15), }, + smoke_failure: None, api_key: None, notes: Vec::new(), } @@ -1160,4 +1265,174 @@ mod tests { "an OOM line beyond the shared tail budget must not surface the note: {notes:?}" ); } + + fn failed_summary(reason: &str) -> DeploymentSummary { + let mut summary = base_summary(); + summary.metrics = SmokeMetrics::default(); + summary.smoke_failure = Some(reason.to_owned()); + summary + } + + #[test] + fn a_failed_smoke_test_is_not_presented_as_ready_and_says_why() { + let rendered = render_summary(&failed_summary("connection closed mid-response")); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("status started, smoke test failed")); + assert!(!rendered.contains("status ready")); + assert!(rendered.contains("time to first token n/a")); + assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains("connection closed mid-response")); + // That the named commands exist and parse is + // `the_commands_the_failure_note_names_exist_and_parse`. + assert!(rendered.contains("check `rocm logs --service vllm-qwen-1720000000`")); + // The hedge: the note must not claim the server is broken, only that it + // may be unable to serve. + assert!(rendered.contains("; the server may not be able to serve requests")); + } + + #[test] + fn a_passing_or_skipped_smoke_test_adds_no_failure_note() { + let passed = render_summary(&base_summary()); + assert!(passed.starts_with("Deployment summary\n")); + assert!(!passed.contains("smoke test")); + // `--no-smoke-test`: default metrics, no failure recorded. + let mut skipped = base_summary(); + skipped.metrics = SmokeMetrics::default(); + let skipped = render_summary(&skipped); + assert!(skipped.starts_with("Deployment summary\n")); + assert!(!skipped.contains("smoke test")); + assert!(skipped.contains("status ready")); + } + + #[test] + fn an_already_running_summary_ignores_a_smoke_failure() { + let mut summary = failed_summary("boom"); + summary.already_running = true; + let rendered = render_summary(&summary); + assert!(rendered.starts_with("Deployment summary (already running)\n")); + assert!(!rendered.contains("smoke test")); + } + + #[test] + fn smoke_outcome_splits_into_summary_fields() { + let metrics = SmokeMetrics { + ttft: Some(Duration::from_millis(5)), + gen_tps: None, + }; + assert_eq!( + SmokeOutcome::Measured(metrics.clone()).into_parts(), + (metrics, None) + ); + assert_eq!( + SmokeOutcome::Failed("x".to_owned()).into_parts(), + (SmokeMetrics::default(), Some("x".to_owned())) + ); + } + + #[test] + fn failure_reason_is_one_line_without_control_characters_and_capped() { + let reason = one_line_reason("HTTP 500\r\n{\"error\":\t\"boom\"}\u{1b}[31m red\u{7}\n\n"); + assert!( + !reason.chars().any(char::is_control), + "a control character survived" + ); + assert_eq!(reason, "HTTP 500 {\"error\": \"boom\"} red"); + let long = one_line_reason(&"x".repeat(5000)); + assert_eq!(long.chars().count(), SMOKE_FAILURE_MAX_CHARS + 3); + assert!(long.ends_with("...")); + assert_eq!(one_line_reason("short"), "short"); + } + + #[test] + fn failure_reason_cap_holds_at_its_boundary() { + // Exactly the cap is kept whole; one more is cut and marked. + let at_cap = "y".repeat(SMOKE_FAILURE_MAX_CHARS); + assert!( + one_line_reason(&at_cap) == at_cap, + "a reason of exactly the cap must be kept unchanged" + ); + let over = "y".repeat(SMOKE_FAILURE_MAX_CHARS + 1); + assert!( + one_line_reason(&over) == format!("{at_cap}..."), + "a reason one character over the cap must be cut to the cap plus an ellipsis" + ); + } + + #[test] + fn failure_reason_cannot_add_a_backtick_quoted_command_to_the_note() { + // A chunk-size line the engine sent is quoted in the provider's error. + let reason = one_line_reason("invalid local provider chunk size `run `rocm stop` now`"); + assert!(!reason.contains('`')); + assert_eq!( + reason, + "invalid local provider chunk size 'run 'rocm stop' now'" + ); + let rendered = render_summary(&failed_summary(&reason)); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + // Constant message: `note` derives from a type that carries an API key. + assert!( + note.split('`').skip(1).step_by(2).count() == 1, + "the note must hold exactly one backtick-quoted command" + ); + } + + #[test] + fn failure_reason_keeps_the_text_after_a_cut_off_escape_sequence() { + // A capture cut mid-sequence must not eat the first real character after + // it: the line break has to reach the escape scanner as a break, not as + // a space it would read as part of the sequence. + for (raw, expected) in [ + ("\u{1b}[31\nHTTP 500", "HTTP 500"), + ("\u{1b}\nHTTP 500", "HTTP 500"), + ("\u{1b}[31mHTTP 500", "HTTP 500"), + ("\u{1b}[31m\nHTTP 500", "HTTP 500"), + ("HTTP\r\n500", "HTTP 500"), + ] { + assert!( + one_line_reason(raw) == expected, + "an escape sequence in the engine's message ate or kept the wrong text" + ); + } + } + + #[test] + fn failure_reason_drops_unicode_format_characters() { + // U+202E (right-to-left override) reorders how the rest of the line + // renders; U+200B and U+FEFF are invisible; U+2028 breaks the row. None + // is matched by `char::is_control`. + let reason = one_line_reason("evil\u{202e}txt\u{200b}.\u{feff}exe\u{2028}next"); + assert_eq!(reason, "eviltxt.exe next"); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format) + ); + } + + #[test] + fn the_commands_the_failure_note_names_exist_and_parse() { + use clap::Parser as _; + let rendered = render_summary(&failed_summary("boom")); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + let commands: Vec<&str> = note.split('`').skip(1).step_by(2).collect(); + // Constant messages only: `commands` is derived from `render_summary`, + // whose input type carries an API key field, and echoing that output + // trips CodeQL's `rust/cleartext-logging`. + assert!( + commands == ["rocm logs --service vllm-qwen-1720000000"], + "the failure note must name exactly the log command" + ); + for command in commands { + assert!( + crate::Cli::try_parse_from(command.split_whitespace()).is_ok(), + "a command named in the failure note is not one rocm accepts" + ); + } + } } diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index d2e8410696..4767b24ea1 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -273,7 +273,7 @@ fn stop_internal_managed_service_confirmed_with( record.stop_requested_unix_ms = Some(requested_at); } record.write()?; - // Drop the endpoint key with the service by deleting its 0600 key file. + // Drop the endpoint key with the service by deleting its owner-only key file. // Best-effort — a stopped service must not fail to stop just because key // cleanup did. // @@ -612,9 +612,10 @@ fn restart_internal_managed_service_inner( if let Some(lease) = port_lease { lease.release(); } - // A restarted public service comes back authenticated: its 0600 key file - // persists across restarts (removed only on stop), so hand the child the same - // file and reuse the key for the readiness probe. + // A restarted public service comes back authenticated: the confirmed stop above + // deleted its owner-only key file, and the key captured before that stop was + // stored back into a fresh one before this point, so hand the child that file + // and reuse the same key for the readiness probe. let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, &record.service_id); let endpoint_api_key = endpoint_keys::endpoint_api_key(paths, &record.service_id); #[cfg(windows)] diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index ae5c2fdb5e..7d0e491063 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -42,8 +42,8 @@ //! failed launch, and the reuse refusals. //! - `spawn`: the detached engine child, its process and stdio plumbing, and //! `ManagedLaunchReport`. -//! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection -//! seam. +//! - `readiness`: waiting on a service, its liveness, and the fault-injection +//! seam (an OOM launch, or a ready launch). //! - `start`: the background launch path and its plain output. //! - `helper_autostart`: starting the background automation helper. //! - `attach`: the attached run and its log streaming. diff --git a/apps/rocm/src/service_launch/endpoint_auth.rs b/apps/rocm/src/service_launch/endpoint_auth.rs index 6a647314bc..2de5247a0e 100644 --- a/apps/rocm/src/service_launch/endpoint_auth.rs +++ b/apps/rocm/src/service_launch/endpoint_auth.rs @@ -98,8 +98,9 @@ pub(crate) fn drop_orphaned_endpoint_key_on_already_running( /// The one such case today: Windows managed Lemonade. Its server reads the /// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn /// primitive only carries path-valued env overrides, so the key never reaches it. -/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds -/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter +/// The vllm engine enforces auth wherever it runs (`VLLM_API_KEY` plus the +/// every-route middleware in `engines/vllm/src/endpoint_auth.rs`), and loopback +/// binds (`public_bind == false`) need no key — both pass. `is_windows` is a parameter /// so both branches are unit-testable off-Windows. pub(crate) fn ensure_public_bind_engine_supported( engine: &str, diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 1e14c85599..c10a29eea6 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -4,12 +4,13 @@ //! Waiting on a launched or already-live managed service: the readiness budget, //! liveness of the tracked process, how a wait ends, and the end-to-end OOM -//! fault-injection seam that fabricates a launch that never became ready. +//! fault-injection seam that fabricates a launch: one that never became ready +//! (it ran out of GPU memory), or, in the `ready` mode, one that became ready. // There is deliberately no `use std::fs`, `use std::thread` or // `use rocm_core::process_is_running` here: this file names them by full path. // Their use sites sit under different `cfg` combinations (the Linux-only zombie -// probe, the OOM fault-injection seam, one Linux-only test), so any single import +// probe, the fault-injection seam, one Linux-only test), so any single import // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. @@ -180,9 +181,22 @@ pub(super) fn record_wait_outcome( /// out-of-memory traceback to the log it owns, and never became ready — so the /// positive OOM-guidance summary path can be verified on a GPU-less CI host, /// where a real launch cannot run. See [`simulate_oom_managed_launch`]. +/// +/// The value `ready` selects the one other mode: the fabricated launch is a +/// managed vLLM launch that *did* become ready (nothing to blame on memory), so a +/// scenario can point its port at a mock engine and watch what the summary does +/// when the post-launch smoke test then fails. Both values open the same seams +/// (the GPU pre-flight waiver and the background-helper suppression). #[cfg(feature = "e2e-oom-fault-injection")] pub(crate) fn e2e_simulate_oom_launch() -> bool { - std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH") + .is_some_and(|value| value == "1" || value == "ready") +} + +/// Whether the armed hook is in its `ready` mode (see [`e2e_simulate_oom_launch`]). +#[cfg(feature = "e2e-oom-fault-injection")] +fn e2e_simulate_ready_launch() -> bool { + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "ready") } /// Release builds carry no fault-injection switch: the seam compiles out @@ -193,12 +207,18 @@ pub(crate) const fn e2e_simulate_oom_launch() -> bool { } /// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a -/// managed vLLM launch that spawned, OOM'd, and never became ready, then return -/// the launch report the summary path would see for it — `already_running: false` -/// (this invocation launched it), `status: "starting"` (it failed to become -/// ready), and a `log_path` whose contents carry a real allocator OOM signature. -/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is -/// spawned. Compiled out of shipped binaries. +/// managed vLLM launch, then return the launch report the summary path would see +/// for it. `already_running` is false either way (this invocation launched it) +/// and no process is spawned. Compiled out of shipped binaries. +/// +/// - Default mode (`ROCM_E2E_SIMULATE_OOM_LAUNCH=1`): the launch OOM'd and never +/// became ready — `status: "starting"` and a `log_path` whose contents carry a +/// real allocator OOM signature. +/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob +/// guidance. +/// - `ready` mode: the launch became ready — `status: "ready"`, the record marked +/// inference-verified, and `log_path: None` (nothing failed to start), so the +/// caller's smoke test runs against whatever answers on `host:port`. #[cfg(feature = "e2e-oom-fault-injection")] #[allow(clippy::too_many_arguments)] pub(super) fn simulate_oom_managed_launch( @@ -231,6 +251,25 @@ pub(super) fn simulate_oom_managed_launch( Some(device_policy_name(device_policy).to_owned()), ); record.gpu_indices = gpu_indices.to_vec(); + if e2e_simulate_ready_launch() { + // The `ready` mode: a launch that came up. The record is what the local + // provider needs to send the smoke test to `host:port`; no log is written + // because nothing here failed to start. + record.status = "ready".to_owned(); + // Load-bearing: without it the smoke test's `/v1/models` listing probe + // fails and the reason becomes "no ready managed service". + record.inference_verified_at_unix_ms = Some(1); + record.write()?; + return Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url: format!("{}/v1", format_http_base_url(host, port)), + status: "ready".to_owned(), + already_running: false, + child_pid: None, + log_path: None, + manifest_path: Some(record.manifest_path), + }); + } record.status = "starting".to_owned(); record.write()?; // A real allocator OOM signature in the log this invocation owns, so the diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 2220426374..4681d89a01 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -11,8 +11,11 @@ //! `prune_managed_service_records`) together with the age gate behind them //! (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). //! -//! Mechanically relocated from `main.rs` with no behavior change; see -//! `docs/architecture.md`. This module reads record state through +//! Mechanically relocated from `main.rs` with no behavior change at the time; +//! see `docs/architecture.md`. Since then both commands also delete the copy +//! of a record's endpoint key an interrupted save left beside it +//! (`staged_endpoint_keys_by_service`), and prune sweeps such a copy whose +//! record is gone. This module reads record state through //! `managed_service_status.rs` and `services_text.rs` (`load_managed_service`, //! `validate_service_id`) and builds its advice text with `service_hints.rs`; //! none of those modules depends on this one. `services_cmd.rs`'s `services()` @@ -72,17 +75,22 @@ struct ServiceRecordArtifacts { log: PathBuf, engine_state: PathBuf, endpoint_key: PathBuf, + /// Copies of the endpoint key a store killed between staging and + /// publishing left beside it, found when the artifacts are listed. + staged_endpoint_keys: Vec, } impl ServiceRecordArtifacts { /// Every artifact, in the order they are reported and deleted. - fn paths(&self) -> [&Path; 4] { + fn paths(&self) -> impl Iterator { [ - &self.manifest, - &self.log, - &self.engine_state, - &self.endpoint_key, + self.manifest.as_path(), + self.log.as_path(), + self.engine_state.as_path(), + self.endpoint_key.as_path(), ] + .into_iter() + .chain(self.staged_endpoint_keys.iter().map(PathBuf::as_path)) } } @@ -102,10 +110,14 @@ fn validate_engine_component(engine: &str) -> Result<()> { Ok(()) } +/// `staged_keys` is [`staged_endpoint_keys_by_service`], taken once by the +/// caller: `prune` builds artifacts for every record, and listing the folder +/// per record would make the plan quadratic in the number of records. fn service_record_artifacts( paths: &AppPaths, service_id: &str, engine: &str, + staged_keys: &HashMap>, ) -> Result { validate_service_id(service_id)?; validate_engine_component(engine)?; @@ -114,9 +126,40 @@ fn service_record_artifacts( log: paths.service_log_path(service_id), engine_state: paths.service_engine_state_path(engine, service_id), endpoint_key: endpoint_keys::endpoint_key_file_path(paths, service_id), + staged_endpoint_keys: staged_keys.get(service_id).cloned().unwrap_or_default(), }) } +/// The service a leftover staged copy of an endpoint key belongs to: `svc` for +/// `svc.endpoint-key.tmp---`, the name a store killed +/// between staging and publishing leaves. `None` for any other file. +fn staged_endpoint_key_owner(file_name: &str) -> Option<&str> { + rocm_core::atomic_write::staged_destination_name(file_name)?.strip_suffix(".endpoint-key") +} + +/// Every leftover staged endpoint-key copy in the services folder, keyed by the +/// service it belongs to, from one listing of the folder. Empty when the folder +/// cannot be listed, in which case those copies are left in place. +fn staged_endpoint_keys_by_service(paths: &AppPaths) -> HashMap> { + let mut staged: HashMap> = HashMap::new(); + let Ok(entries) = fs::read_dir(paths.services_dir()) else { + return staged; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if let Some(owner) = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(staged_endpoint_key_owner) + { + staged.entry(owner.to_owned()).or_default().push(path); + } + } + for copies in staged.values_mut() { + copies.sort(); + } + staged +} + /// Delete every artifact that exists, collecting failures instead of stopping at /// the first one. /// @@ -164,7 +207,7 @@ fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result /// state file, and `refresh_managed_service_runtime_liveness` then returns early /// for any non-live status without ever consulting the recorded pids — so a /// server whose engine reported failure while its process is still up reads as -/// removable, and deleting it takes the log and the 0600 endpoint key out from +/// removable, and deleting it takes the log and the owner-only endpoint key out from /// under a process that may still be serving. /// /// A recorded process that could not be queried (on Windows, a live one this @@ -308,7 +351,12 @@ pub(crate) fn remove_managed_service_record_with( bail!("Removing local server record `{service_id}` requires --yes.\n\nTry: {hint}"); } - let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; + let artifacts = service_record_artifacts( + paths, + &record.service_id, + &record.engine, + &staged_endpoint_keys_by_service(paths), + )?; let mut output = String::new(); // Printed *before* the delete, because after it there is nothing left to // point at: this is the last moment the log path is useful. @@ -465,7 +513,7 @@ fn describe_hours(hours: u64) -> String { /// reason. /// /// "No `.json` beside it" is also exactly what a *launch in progress* looks -/// like: `serve` writes the 0600 `.endpoint-key` before +/// like: `serve` writes the owner-only `.endpoint-key` before /// `spawn_managed_engine_child` (in `service_launch/spawn.rs`) writes the first /// `.json`. Nothing about /// the file distinguishes the two cases — under `--any-age` there is no age gate @@ -473,19 +521,20 @@ fn describe_hours(hours: u64) -> String { /// managed-launch lock held, which excludes that window outright; see /// [`prune_managed_service_records`]. /// +/// A staged copy of an endpoint key (`.endpoint-key.tmp-…`, left by a store +/// killed before it published) is judged the same way, by `.json`: it holds +/// the key in plaintext, and nothing else would ever remove it. +/// /// `services_dir` also holds `launch.lock`, which is shared by every managed /// launch rather than owned by one service (see -/// [`AppPaths::managed_launch_lock_path`]). Only the three per-service -/// extensions are considered, so the lock is never a candidate — including the -/// one this sweep is itself running under. +/// [`AppPaths::managed_launch_lock_path`]). Only the per-service extensions and +/// staged endpoint keys are considered, so the lock is never a candidate — +/// including the one this sweep is itself running under. fn collect_service_orphans(paths: &AppPaths, min_age: Duration, now: SystemTime) -> Vec { let services_dir = paths.services_dir(); let mut orphans = Vec::new(); - let push_if_orphaned = |path: &Path, orphans: &mut Vec| { - let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { - return; - }; + let push_if_orphaned = |path: &Path, stem: &str, orphans: &mut Vec| { if services_dir.join(format!("{stem}.json")).exists() { return; } @@ -500,7 +549,15 @@ fn collect_service_orphans(paths: &AppPaths, min_age: Duration, now: SystemTime) path.extension().and_then(|value| value.to_str()), Some("log" | "endpoint-key") ) { - push_if_orphaned(&path, &mut orphans); + if let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) { + push_if_orphaned(&path, stem, &mut orphans); + } + } else if let Some(stem) = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(staged_endpoint_key_owner) + { + push_if_orphaned(&path, stem, &mut orphans); } } } @@ -516,8 +573,10 @@ fn collect_service_orphans(paths: &AppPaths, min_age: Duration, now: SystemTime) continue; }; for path in states.flatten().map(|entry| entry.path()) { - if path.extension().and_then(|value| value.to_str()) == Some("json") { - push_if_orphaned(&path, &mut orphans); + if path.extension().and_then(|value| value.to_str()) == Some("json") + && let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) + { + push_if_orphaned(&path, stem, &mut orphans); } } } @@ -543,6 +602,7 @@ fn build_service_prune_plan( // `load_managed_services` refreshes each record against the engine state and // the real processes before returning it, so liveness below is read from the // refreshed view, never from the manifest as it was on disk. + let staged_keys = staged_endpoint_keys_by_service(paths); for record in load_managed_services(paths)? { if managed_service_record_is_in_use_with(&record, probe) { plan.skipped_live += 1; @@ -568,13 +628,15 @@ fn build_service_prune_plan( plan.skipped.push(skipped); continue; } - let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { - Ok(artifacts) => artifacts, - Err(error) => { - plan.skipped.push(format!("{}: {error}", record.service_id)); - continue; - } - }; + let artifacts = + match service_record_artifacts(paths, &record.service_id, &record.engine, &staged_keys) + { + Ok(artifacts) => artifacts, + Err(error) => { + plan.skipped.push(format!("{}: {error}", record.service_id)); + continue; + } + }; if !prunable_by_modified( manifest_times.get(&artifacts.manifest).copied(), min_age, @@ -730,7 +792,7 @@ fn apply_service_prune_plan_with( for entry in &plan.remove { // Liveness was snapshotted while the plan was built. A // `rocm services restart --yes` landing in that window would have - // its log and 0600 endpoint key deleted out from under a live process, + // its log and owner-only endpoint key deleted out from under a live process, // so re-read the record immediately before touching its files — // `remove` narrows the same window by loading the record it deletes. // Neither closes it: the check and the delete are not atomic either way. @@ -774,7 +836,7 @@ fn apply_service_prune_plan_with( /// /// [`collect_service_orphans`] calls a file with no `.json` beside it a /// leftover, and that is also what a launch looks like mid-flight: `serve` -/// writes the 0600 `.endpoint-key` at the top of the managed path, and the +/// writes the owner-only `.endpoint-key` at the top of the managed path, and the /// first `.json` only lands at `record.write()` inside /// `spawn_managed_engine_child` (in `service_launch/spawn.rs`). Between those two /// writes a live server's secret is indistinguishable from a leftover, and @@ -1116,8 +1178,8 @@ mod tests { let (root, paths) = test_paths("services-remove-traversal"); paths.ensure()?; - let by_engine = service_record_artifacts(&paths, "svc-ok", "../../../etc"); - let by_id = service_record_artifacts(&paths, "../../etc/passwd", "vllm"); + let by_engine = service_record_artifacts(&paths, "svc-ok", "../../../etc", &HashMap::new()); + let by_id = service_record_artifacts(&paths, "../../etc/passwd", "vllm", &HashMap::new()); let _ = fs::remove_dir_all(&root); assert!( @@ -1176,6 +1238,145 @@ mod tests { Ok(()) } + /// A store killed between staging the key and publishing it leaves a + /// second, plaintext copy beside the key. Removing the record takes it + /// too, and leaves another service's staged copy alone. + #[test] + fn services_remove_takes_a_staged_copy_of_the_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("services-remove-staged-key"); + paths.ensure()?; + plant_service_record(&paths, "svc-remove-me", "failed", 999_999_999)?; + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-remove-me"); + let staged = rocm_core::atomic_write::stage_owner_only_file_unpublished(&key, b"secret")?; + let other_key = endpoint_keys::endpoint_key_file_path(&paths, "svc-remove-me-too"); + let other_staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&other_key, b"other")?; + + let rendered = remove_managed_service_record(&paths, "svc-remove-me", true); + let staged_exists = staged.exists(); + let other_staged_exists = other_staged.exists(); + let _ = fs::remove_dir_all(&root); + + let rendered = rendered?; + assert!(!staged_exists, "the staged key copy must be deleted"); + assert!( + other_staged_exists, + "another service's staged key must be left" + ); + assert!(rendered.contains(" files removed: 5"), "{rendered}"); + Ok(()) + } + + /// Pruning a record that is still on disk takes the staged key copy an + /// interrupted save left beside its key, as `remove` does. + #[test] + fn services_prune_takes_a_pruned_records_staged_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("services-prune-record-staged-key"); + paths.ensure()?; + plant_service_record(&paths, "svc-pruned", "failed", 999_999_999)?; + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-pruned"); + let staged = rocm_core::atomic_write::stage_owner_only_file_unpublished(&key, b"secret")?; + + let outcome = prune_managed_service_records(&paths, 0, false, true); + let record_exists = paths.service_manifest_path("svc-pruned").exists(); + let staged_exists = staged.exists(); + let _ = fs::remove_dir_all(&root); + + let outcome = outcome?; + assert!( + !record_exists, + "premise: prune must remove the record:\n{}", + outcome.text + ); + assert!( + !staged_exists, + "the pruned record's staged key copy must go with it:\n{}", + outcome.text + ); + Ok(()) + } + + /// A staged key copy whose record is gone is an orphan like the key itself; + /// one whose record is still there is not. + #[test] + fn services_prune_sweeps_a_staged_endpoint_key_whose_record_is_gone() -> Result<()> { + let (root, paths) = test_paths("services-prune-staged-key"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-orphaned", "failed", 999_999_999)?; + fs::remove_file(&record.manifest_path)?; + let orphan_key = endpoint_keys::endpoint_key_file_path(&paths, "svc-orphaned"); + let orphan_staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&orphan_key, b"secret")?; + // A record whose server is still running keeps its staged copy: the + // sweep only takes what no record owns. + plant_service_record(&paths, "svc-kept", "ready", std::process::id())?; + let kept_key = endpoint_keys::endpoint_key_file_path(&paths, "svc-kept"); + let kept_staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&kept_key, b"kept")?; + + let outcome = prune_managed_service_records(&paths, 0, false, true); + let orphan_staged_exists = orphan_staged.exists(); + let kept_staged_exists = kept_staged.exists(); + let _ = fs::remove_dir_all(&root); + + let outcome = outcome?; + assert!( + !orphan_staged_exists, + "the orphaned staged key must be swept:\n{}", + outcome.text + ); + assert!( + kept_staged_exists, + "a live record's staged key must be left" + ); + Ok(()) + } + + /// The sweep takes only staged endpoint-key copies: a record that was + /// being rewritten when its writer died leaves `.json.tmp-…`, which is + /// not a key and is not the sweep's to judge. + #[test] + fn the_orphan_sweep_leaves_a_half_written_manifest_alone() -> Result<()> { + let (root, paths) = test_paths("orphans-manifest-tmp"); + paths.ensure()?; + let manifest = paths.service_manifest_path("svc-mid-write"); + let staged_manifest = + rocm_core::atomic_write::stage_file_for_atomic_publish(&manifest, b"{}")?; + + let orphans = collect_service_orphans(&paths, Duration::ZERO, SystemTime::now()); + let _ = fs::remove_dir_all(&root); + assert!( + !orphans.contains(&staged_manifest), + "a staged manifest was swept: {orphans:?}" + ); + Ok(()) + } + + /// A staged key copy goes through the same age gate as every other + /// leftover: one written just now is kept under a non-zero age, and taken + /// once the gate is lifted. + #[test] + fn the_age_gate_applies_to_a_staged_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("orphans-staged-key-age"); + paths.ensure()?; + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-fresh"); + let staged = rocm_core::atomic_write::stage_owner_only_file_unpublished(&key, b"secret")?; + let now = SystemTime::now(); + + let gated = collect_service_orphans(&paths, Duration::from_hours(24), now); + let ungated = collect_service_orphans(&paths, Duration::ZERO, now); + let _ = fs::remove_dir_all(&root); + assert!( + !gated.contains(&staged), + "a fresh staged key was swept: {gated:?}" + ); + assert!( + ungated.contains(&staged), + "a staged orphan key was missed: {ungated:?}" + ); + Ok(()) + } + /// The engine state file lives outside `services_dir`, so hand-deleting the /// manifest and log — the workaround this command replaces — leaves it /// behind forever. Prune has to sweep it. @@ -1217,7 +1418,7 @@ mod tests { } /// The window the launch lock exists to close here: `serve` writes - /// the 0600 endpoint key *before* the first manifest, so between those two + /// the owner-only endpoint key *before* the first manifest, so between those two /// writes a live server's key has no `.json` beside it — the exact shape /// the sweep calls a leftover, and `--any-age` leaves no age gate to hide it /// behind. `serve` holds the managed-launch lock across the whole of that @@ -1656,7 +1857,7 @@ mod tests { /// adopts `failed` straight from the engine's own state file and the /// liveness refresh then returns early for a non-live status, so a server /// whose engine reported failure while its process is still up would be - /// removable — taking the log and the 0600 endpoint key of a live process. + /// removable — taking the log and the owner-only endpoint key of a live process. #[test] fn services_removal_refuses_a_failed_record_whose_process_is_still_alive() -> Result<()> { let (root, paths) = test_paths("services-remove-live-pid"); @@ -1778,7 +1979,7 @@ mod tests { /// The window the pre-delete liveness re-check exists to narrow: the plan is /// built from a snapshot, and a `rocm services restart --yes` landing /// between that snapshot and the delete would otherwise have its log and its - /// 0600 endpoint key deleted out from under a serving process. + /// owner-only endpoint key deleted out from under a serving process. /// /// Driven through [`apply_service_prune_plan`] with a hand-built plan /// because the window cannot be opened from outside: `prune` builds and @@ -1794,7 +1995,12 @@ mod tests { let record = plant_service_record(&paths, "svc-relaunched", "ready", std::process::id())?; // The entry a plan built moments earlier, while the record was still // stopped, would carry into the delete loop. - let artifacts = match service_record_artifacts(&paths, "svc-relaunched", "vllm") { + let artifacts = match service_record_artifacts( + &paths, + "svc-relaunched", + "vllm", + &staged_endpoint_keys_by_service(&paths), + ) { Ok(artifacts) => artifacts, Err(error) => { let _ = fs::remove_dir_all(&root); @@ -1830,7 +2036,7 @@ mod tests { ); assert!( key_exists, - "the 0600 endpoint key of a live process must survive:\n{}", + "the owner-only endpoint key of a live process must survive:\n{}", outcome.text ); assert_eq!(outcome.removed_records, 0); diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 804e26a093..be13d9c63d 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -693,6 +693,7 @@ fn install_wheel_runtime( probe_rocm_sdk_runtime_for_target(&env_python, Some(device_target.as_str())) .context("TheRock packages did not expose a usable rocm_sdk runtime")?; validate_rocm_sdk_runtime_probe(&rocm_sdk_probe)?; + report_runtime_librocdxg_link(&mut output, &rocm_sdk_probe); let installed_version = rocm_sdk_probe .rocm_sdk_version .clone() @@ -752,6 +753,50 @@ fn install_wheel_runtime( Ok(SdkInstallResult::installed(output)) } +/// On WSL the runtime's HSA runtime opens `librocdxg.so`, which the wheel does +/// not ship, so it would load the host's copy instead of the one beside it. Link +/// the runtime's own copy into the library directories the probe recorded and +/// write one report line per copy. The WSL gate is read inside +/// `link_installed_runtime_copies`, so the install cannot pass a wrong answer. +fn report_runtime_librocdxg_link(output: &mut String, probe: &RocmSdkPythonProbe) { + write_librocdxg_report( + output, + rocm_core::runtime_rocdxg::link_installed_runtime_copies(&librocdxg_dirs(probe)), + ); +} + +/// [`report_runtime_librocdxg_link`] with the WSL gate supplied, so the unit tests +/// run the WSL branch on every Unix host. Shares the directory choice and the report +/// writing with the install's own path. +#[cfg(all(test, unix))] +fn report_runtime_librocdxg_link_when( + output: &mut String, + is_wsl: bool, + probe: &RocmSdkPythonProbe, +) { + write_librocdxg_report( + output, + rocm_core::runtime_rocdxg::link_installed_runtime_copies_when( + is_wsl, + &librocdxg_dirs(probe), + ), + ); +} + +/// The directories whose `librocdxg.so.1` is linked: the ones the probe recorded. +/// `rocm examine` scans a list computed separately from the registry record +/// (`managed_runtime_library_dirs`) that includes these; a test in `rocm-core` +/// pins that. +fn librocdxg_dirs(probe: &RocmSdkPythonProbe) -> Vec { + probe.library_paths.clone() +} + +fn write_librocdxg_report(output: &mut String, lines: Vec) { + for line in lines { + let _ = writeln!(output, " {line}"); + } +} + /// The host's legacy/system ROCm version when it is strictly newer than the /// version rocm-cli is about to install, otherwise `None`. Used to explain why a /// seemingly older wheel version is selected over the host's ROCm. @@ -1139,6 +1184,49 @@ mod tests { use super::test_support::{test_paths, test_runtime_manifest}; use super::*; + /// The install report links the probe's library directories when the host is + /// WSL, and touches nothing otherwise. Driven with the gate supplied, so both + /// branches run on every Unix host; the real gate is tested in + /// `runtime_rocdxg::tests` through a simulated host root (hooks lane). + #[cfg(unix)] + #[test] + fn the_install_report_links_only_the_probes_library_directories_when_wsl() { + let dir = tempfile::tempdir().expect("temp dir"); + let lib = dir.path().join("lib"); + let elsewhere = dir.path().join("elsewhere"); + for d in [&lib, &elsewhere] { + fs::create_dir_all(d).expect("dir"); + fs::write(d.join("libhsa-runtime64.so.1"), "").expect("runtime"); + fs::write(d.join("librocdxg.so.1"), "own").expect("copy"); + } + let probe = RocmSdkPythonProbe { + library_paths: vec![lib.clone()], + root_path: Some(elsewhere.clone()), + ..RocmSdkPythonProbe::default() + }; + + let mut off = String::new(); + report_runtime_librocdxg_link_when(&mut off, false, &probe); + assert!(off.is_empty(), "{off}"); + assert!(lib.join("librocdxg.so").symlink_metadata().is_err()); + + let mut on = String::new(); + report_runtime_librocdxg_link_when(&mut on, true, &probe); + // The whole line, indent included: a copy path or a lost indent changes it. + assert!( + on.starts_with(&format!( + " librocdxg: linked {} to the runtime's own copy", + lib.join("librocdxg.so").display() + )) && on.ends_with('\n'), + "{on}" + ); + assert_eq!(fs::read(lib.join("librocdxg.so")).expect("link"), b"own"); + assert!( + elsewhere.join("librocdxg.so").symlink_metadata().is_err(), + "only the probe's library paths are linked" + ); + } + /// `--format tarball` installs no Python, so a Python choice cannot be honoured /// and is refused before anything is resolved or downloaded. The refusal comes /// ahead of the platform's tarball check so it reads the same on every host. @@ -1491,6 +1579,74 @@ mod tests { } } + /// Pins the production derivation to the literal keys of the side-by-side + /// e2e fixture, from the fixture's inputs. The fixture + /// (tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs) re-derives + /// keys in `Role::key` to plant runtimes without installing them, and its + /// planting step checks those against the same literals in + /// `Role::expected_key`. So a change here fails this test, and a change to + /// the fixture fails the scenario's Given. + #[test] + fn wheel_runtime_key_matches_the_side_by_side_e2e_fixture() { + let composition = |generation: &str, specs: [&str; 4]| WheelRuntimeComposition { + source_layout_generation: generation.to_owned(), + package_specs: specs.iter().map(|spec| (*spec).to_owned()).collect(), + rocm_sdk_target: Some("gfx942".to_owned()), + }; + let cases = [ + ( + TheRockChannel::Release, + "7.14.0", + composition( + "multi-arch-v2", + [ + "rocm[libraries,devel,device-gfx942]==7.14.0", + "torch[device-gfx942]==2.11.0+rocm7.14.0", + "torchvision[device-gfx942]==0.26.0+rocm7.14.0", + "torchaudio==2.11.0+rocm7.14.0", + ], + ), + "release-wheel-multi-arch-7-14-0-992a08553d8be8f0", + ), + ( + TheRockChannel::Release, + "10.0.0", + composition( + "next-v1", + [ + "rocm[libraries,device-gfx942]==10.0.0", + "torch[device-gfx942]==2.13.0+rocm10.0.0", + "torchvision[device-gfx942]==0.28.0+rocm10.0.0", + "torchaudio==2.11.0.2+rocm10.0.0", + ], + ), + "release-wheel-multi-arch-10-0-0-3d0e256f664a960d", + ), + ( + TheRockChannel::Nightly, + "10.1.0a20260822", + composition( + "multi-arch-v2", + [ + "rocm[libraries,device-gfx942]==10.1.0a20260822", + "torch[device-gfx942]==2.12.0+rocm10.1.0a20260822", + "torchvision[device-gfx942]==0.27.0+rocm10.1.0a20260822", + "torchaudio==2.12.0+rocm10.1.0a20260822", + ], + ), + "nightly-wheel-multi-arch-10-1-0a20260822-95e3a823499c530b", + ), + ]; + for (channel, version, composition, expected) in cases { + assert_eq!( + wheel_runtime_key(channel, version, &composition), + expected, + "the production key derivation changed; update these literals, and `Role::key` \ + and `Role::expected_key` in runtime_side_by_side_steps.rs, together" + ); + } + } + #[test] fn aggregate_wheel_runtime_key_separates_device_payloads_at_one_version() { let base = WheelRuntimeComposition { diff --git a/apps/rocm/src/therock/probe.rs b/apps/rocm/src/therock/probe.rs index abe86ea719..11c5bb3936 100644 --- a/apps/rocm/src/therock/probe.rs +++ b/apps/rocm/src/therock/probe.rs @@ -89,7 +89,8 @@ pub(crate) struct RuntimeDeviceProbe { /// /// `library_paths` must be the runtime's recorded ROCm library directories (see /// [`RocmSdkPythonProbe::library_paths`]). They are prepended to -/// `LD_LIBRARY_PATH` for the child, which is how a served process resolves them. +/// `LD_LIBRARY_PATH` for the child, which is how a served process resolves them, +/// followed by the directory holding the amd-smi link when one is needed. /// /// This cannot be replaced with an in-process `rocm_sdk.initialize_process()` /// call in the probe script. Measured on MI300X against a runtime that serves @@ -103,7 +104,10 @@ pub(crate) fn probe_runtime_devices( ) -> Result { let mut env = Vec::new(); if !library_paths.is_empty() { - let mut entries = library_paths.to_vec(); + // The recorded directories plus the amd-smi shim (see + // `rocm_core::ensure_amd_smi_compat`): without it torch reports no devices + // beside a system ROCm although HIP sees them. + let mut entries = rocm_core::runtime_loader_path_entries(python_executable, library_paths); if let Some(existing) = std::env::var_os(RUNTIME_LIBRARY_PATH_ENV) { entries.extend(split_runtime_path(&existing)); } @@ -447,6 +451,45 @@ mod tests { use crate::therock::test_support::test_paths; use std::fs; + /// The device probe has to carry the amd-smi link's directory on its loader + /// path, or a torch beside a host ROCm counts no devices. The interpreter here + /// is a script that reports the `LD_LIBRARY_PATH` it was started with. + #[cfg(target_os = "linux")] + #[test] + fn the_device_probe_runs_with_the_amd_smi_link_on_its_loader_path() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let root = std::env::temp_dir().join(format!( + "rocm-probe-amd-smi-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + )); + let bin = root.join("env").join("bin"); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&bin)?; + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + let python = bin.join("python"); + fs::write( + &python, + "#!/bin/sh\nprintf '{\"import_ok\": true, \"error\": \"%s\"}' \"$LD_LIBRARY_PATH\"\n", + )?; + fs::set_permissions(&python, fs::Permissions::from_mode(0o755))?; + + let probe = probe_runtime_devices(&python, std::slice::from_ref(&sdk_lib))?; + + let loader_path = probe.error.unwrap_or_default(); + let compat_dir = root.join("env").join(rocm_core::RUNTIME_COMPAT_DIR_NAME); + assert!( + loader_path.starts_with(&format!("{}:{}", sdk_lib.display(), compat_dir.display())), + "{loader_path}" + ); + assert!(compat_dir.join("libamd_smi.so").exists()); + fs::remove_dir_all(root).ok(); + Ok(()) + } + #[test] fn parses_rocm_sdk_probe_contract() -> Result<()> { let root_path = if cfg!(windows) { diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index 381929ba0f..33cef9ada7 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -1927,7 +1927,11 @@ mod tests { ); assert_eq!(nightly[1], "torch[device-gfx942]==2.13.*"); - let no_row = published_pip_requirements("10.2.0", "2.12.0+rocm10.2.0", "gfx942", false); + let ten_two = published_pip_requirements("10.2.0", "2.11.0.3", "gfx942", false); + assert_eq!(ten_two[1], "torch[device-gfx942]==2.14.*"); + assert_eq!(ten_two[2], "torchvision[device-gfx942]==0.29.*"); + + let no_row = published_pip_requirements("10.3.0", "2.12.0+rocm10.3.0", "gfx942", false); assert_eq!(no_row[1], "torch[device-gfx942]"); assert_eq!(no_row[2], "torchvision[device-gfx942]"); } diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 1f0972ba3b..98751906cc 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -461,12 +461,7 @@ fn plan_runtime_update( /// a base can be overridden or rehosted, and the recorded generation is the only /// statement about the stream that survives that. fn manifest_source_layout(manifest: &InstalledRuntimeManifest) -> Result { - SourceLayout::from_generation(manifest.source_layout_generation.as_deref().or_else(|| { - manifest - .wheel_composition - .as_ref() - .map(|composition| composition.source_layout_generation.as_str()) - })) + SourceLayout::from_generation(manifest.recorded_source_layout_generation()) } /// The family override to re-resolve a wheel manifest's update *plan* with. diff --git a/apps/rocm/src/tool_policy.rs b/apps/rocm/src/tool_policy.rs index 9634985e45..e32af6f585 100644 --- a/apps/rocm/src/tool_policy.rs +++ b/apps/rocm/src/tool_policy.rs @@ -170,6 +170,18 @@ pub(crate) fn validate_chat_tool_call(call: &providers::ChatToolCall) -> Result< if !call.arguments.is_object() { bail!("ROCm tool `{}` arguments must be a JSON object", call.name); } + // No ROCm tool takes a parameter called `arguments` (held by the + // `no_rocm_tool_declares_an_arguments_parameter` tests in + // `providers/tool_schema.rs` and in `rocm-dash-tui`'s `agent/tools.rs`). + // A call that carries one holds a second payload next to the first, and + // the approval card and the command it approves must never be able to + // read different ones. + if call.arguments.get("arguments").is_some() { + bail!( + "ROCm tool `{}` arguments must not contain a nested `arguments` field", + call.name + ); + } reject_control_characters_in_tool_arguments(&call.name, &call.arguments)?; match call.name.as_str() { "examine" @@ -1111,6 +1123,49 @@ mod tests { } } + #[test] + fn validate_chat_tool_call_refuses_a_nested_arguments_field_for_every_tool() { + // Every name `validate_chat_tool_call` accepts (its first `match` + // arm), so a future change that scoped the nested-`arguments` refusal + // to only some tools would be caught here for all of them, not just + // the mutating ones. + for name in [ + "examine", + "doctor", + "bridge_snapshot", + "gpu_snapshot", + "engines", + "services", + "service_logs", + "automations", + "natural_language_plan", + "path_exists", + "port_status", + "rocm_command", + "update_check", + "install_sdk_dry_run", + "install_sdk", + "install_engine", + "launch_server", + "stop_server", + "watcher_enable", + "watcher_disable", + "proposal_action", + ] { + let call = providers::ChatToolCall { + id: None, + name: name.to_owned(), + arguments: serde_json::json!({ "arguments": {} }), + }; + let err = validate_chat_tool_call(&call) + .expect_err("a nested arguments field must be refused"); + assert!( + err.to_string().contains("nested `arguments`"), + "`{name}`: unexpected error: {err}" + ); + } + } + #[test] fn runtimes_activate_approval_modal_uses_the_runtime_noun() { // `comfyui install`'s selection errors steer the user to `rocm runtimes` diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 70587d7456..96b80de45f 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -27,6 +27,9 @@ tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } [dev-dependencies] +# The leftover a killed owner-only key store leaves, for the tests of the +# cleanups that remove it (`atomic_write::stage_owner_only_file_unpublished`). +rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # The fake engine child both engine IPC callers are tested against. rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs index 76486a9153..73df2dceb3 100644 --- a/apps/rocmd/src/anthropic_proxy.rs +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -47,8 +47,8 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use rocm_core::{ ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, - format_http_base_url, http_get_with_auth, http_post_json_with_auth, merge_json_state_file, - openai_models_endpoint_has_model, + JsonStateFileRefused, format_http_base_url, http_get_with_auth, http_post_json_with_auth, + merge_json_state_file, openai_models_endpoint_has_model, }; use serde_json::{Value, json}; @@ -146,20 +146,66 @@ fn wait_for_engine_listing(config: &AnthropicProxyConfig, upstream: &str) -> any anyhow::bail!("the engine did not list a model in time") } +/// How many times [`publish`] tries to record the endpoint before it gives up. +const PUBLISH_ATTEMPTS: usize = 5; +/// The pause between two of [`publish`]'s attempts. +const PUBLISH_RETRY_PAUSE: Duration = Duration::from_millis(200); + +/// Record the endpoint in the engine's state file. +/// +/// `merge_json_state_file` refuses a state file it cannot merge into rather +/// than replace it. The vLLM and AIM engines still rewrite their state files +/// in place, so a file refused as empty or not valid JSON can be one caught +/// mid-write and whole again a moment later: that refusal is tried again, up +/// to [`PUBLISH_ATTEMPTS`] attempts [`PUBLISH_RETRY_PAUSE`] apart. Any other +/// failure (a whole document that is not an object, or failing to lock, read +/// or write the file) is returned at once. fn publish( config: &AnthropicProxyConfig, url: &str, mode: AnthropicEndpointMode, +) -> anyhow::Result<()> { + publish_with_pause(config, url, mode, std::thread::sleep) +} + +/// [`publish`] with the pause between attempts supplied by the caller, so a +/// test can see each pause and change the file between attempts. +fn publish_with_pause( + config: &AnthropicProxyConfig, + url: &str, + mode: AnthropicEndpointMode, + mut pause: impl FnMut(Duration), ) -> anyhow::Result<()> { // The listener URL carries no `/v1`: Anthropic SDKs append `/v1/messages`. let base = url.trim_end_matches('/').trim_end_matches("/v1"); - merge_json_state_file( - &config.state_path, - &json!({ - ANTHROPIC_ENDPOINT_URL_STATE_KEY: base, - ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), - }), - ) + let patch = json!({ + ANTHROPIC_ENDPOINT_URL_STATE_KEY: base, + ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), + }); + let mut attempt = 1; + loop { + let error = match merge_json_state_file(&config.state_path, &patch) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + let may_be_mid_write = error + .downcast_ref::() + .is_some_and(|refused| refused.refusal.may_be_mid_write()); + if may_be_mid_write && attempt < PUBLISH_ATTEMPTS { + attempt += 1; + pause(PUBLISH_RETRY_PAUSE); + continue; + } + let attempts = if attempt == 1 { + "1 attempt".to_owned() + } else { + format!("{attempt} attempts") + }; + return Err(error.context(format!( + "could not publish the Anthropic endpoint to {} after {attempts}", + config.state_path.display() + ))); + } } #[derive(Clone)] @@ -2269,6 +2315,140 @@ mod tests { ); } + fn publish_config(dir: &std::path::Path) -> AnthropicProxyConfig { + std::fs::create_dir_all(dir).unwrap(); + AnthropicProxyConfig { + host: "127.0.0.1".to_owned(), + engine_port: 1, + served_model: MODEL.to_owned(), + state_path: dir.join("state.json"), + probe_api_key: None, + } + } + + /// A state file caught mid-write by an engine that rewrites it in place is + /// refused, and the publish tries again, after the production pause, once + /// it is whole. + #[test] + fn publish_retries_a_state_file_caught_mid_write() { + let dir = unique_temp_dir("publish-retry"); + let config = publish_config(&dir); + std::fs::write(&config.state_path, b"").unwrap(); + let mut pauses = Vec::new(); + + let result = publish_with_pause( + &config, + "http://127.0.0.1:8000/v1", + AnthropicEndpointMode::Proxy, + |pause| { + pauses.push(pause); + std::fs::write(&config.state_path, br#"{"status": "ready"}"#).unwrap(); + }, + ); + + let state: Value = + serde_json::from_slice(&std::fs::read(&config.state_path).unwrap()).unwrap(); + let _ = std::fs::remove_dir_all(dir); + result.expect("the second attempt publishes"); + assert_eq!(pauses, [Duration::from_millis(200)]); + assert_eq!(state["status"], "ready", "{state}"); + assert_eq!(state["anthropic_endpoint_url"], "http://127.0.0.1:8000"); + assert_eq!(state["anthropic_endpoint_mode"], "proxy"); + } + + /// The production `publish`, against a state file that stays torn: five + /// attempts with a real 200 ms pause between each, so it cannot give up + /// in less than 800 ms; then an error naming the file, which is left as + /// it was. + #[test] + fn publish_gives_up_on_a_state_file_that_stays_torn() { + let dir = unique_temp_dir("publish-gives-up"); + let config = publish_config(&dir); + let torn = br#"{"status": "ready", "server_pid": 4"#; + std::fs::write(&config.state_path, torn).unwrap(); + let started = std::time::Instant::now(); + + let result = publish( + &config, + "http://127.0.0.1:8000/v1", + AnthropicEndpointMode::Proxy, + ); + + let elapsed = started.elapsed(); + let after = std::fs::read(&config.state_path).unwrap(); + let error = format!("{:#}", result.expect_err("a torn file is refused")); + assert!( + error.starts_with(&format!( + "could not publish the Anthropic endpoint to {} after 5 attempts: ", + config.state_path.display() + )), + "{error}" + ); + assert!(error.contains("is not valid JSON"), "{error}"); + let _ = std::fs::remove_dir_all(dir); + assert!( + elapsed >= Duration::from_millis(800), + "four 200 ms pauses between five attempts, took {elapsed:?}" + ); + assert_eq!(after, torn, "the refused file is left as it was"); + } + + /// A failure a mid-write cannot explain is returned at once, without a + /// pause: a whole document that is not an object, and a state file whose + /// lock cannot be taken. + #[test] + fn publish_does_not_retry_a_failure_a_mid_write_cannot_explain() { + let dir = unique_temp_dir("publish-no-retry"); + let config = publish_config(&dir); + std::fs::write(&config.state_path, b"[1, 2]").unwrap(); + // A regular file where the state directory should be, so the lock file + // beside the state file cannot be created. + let blocked = AnthropicProxyConfig { + state_path: dir.join("not-a-dir").join("state.json"), + ..publish_config(&dir) + }; + std::fs::write(dir.join("not-a-dir"), b"").unwrap(); + + let mut pauses = Vec::new(); + let not_an_object = publish_with_pause( + &config, + "http://127.0.0.1:8000/v1", + AnthropicEndpointMode::Proxy, + |pause| pauses.push(pause), + ); + let unlockable = publish_with_pause( + &blocked, + "http://127.0.0.1:8000/v1", + AnthropicEndpointMode::Proxy, + |pause| pauses.push(pause), + ); + + let after = std::fs::read(&config.state_path).unwrap(); + let _ = std::fs::remove_dir_all(dir); + assert!(pauses.is_empty(), "no retry: {pauses:?}"); + let error = format!("{:#}", not_an_object.expect_err("an array is refused")); + assert!( + error.starts_with(&format!( + "could not publish the Anthropic endpoint to {} after 1 attempt: ", + config.state_path.display() + )), + "{error}" + ); + assert!( + error.contains("holds a JSON array, not an object"), + "{error}" + ); + assert_eq!(after, b"[1, 2]", "the refused file is left as it was"); + let error = format!("{:#}", unlockable.expect_err("the lock cannot be taken")); + assert!( + error.starts_with(&format!( + "could not publish the Anthropic endpoint to {} after 1 attempt: ", + blocked.state_path.display() + )), + "{error}" + ); + } + #[test] fn publish_records_the_endpoint_without_a_v1_suffix() { let dir = unique_temp_dir("publish"); diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index eda8243ac2..ae5dbba841 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -424,10 +424,7 @@ mod tests { const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; fn healthcheck_envelope() -> EngineRequestEnvelope { - EngineRequestEnvelope { - method: EngineMethod::Healthcheck, - payload: json!({"service_id": SERVICE_ID}), - } + EngineRequestEnvelope::new(EngineMethod::Healthcheck, json!({"service_id": SERVICE_ID})) } fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { @@ -454,7 +451,11 @@ mod tests { assert_eq!(engine.argv(), ["__engine-stdio", "vllm"]); assert_eq!( engine.stdin(), - "{\"method\":\"healthcheck\",\"payload\":{\"service_id\":\"svc-stdio-child\"}}\n" + format!( + "{{\"protocol_version\":\"{}\",\"method\":\"healthcheck\",\ + \"payload\":{{\"service_id\":\"svc-stdio-child\"}}}}\n", + rocm_engine_protocol::ENGINE_PROTOCOL_VERSION + ) ); } @@ -536,7 +537,9 @@ mod tests { FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); assert_eq!( request(&garbage, &paths).unwrap_err().to_string(), - "failed to parse engine response envelope; stderr: oops" + "failed to parse engine response envelope: the engine's stdout is not JSON \ + (expected ident at line 1 column 2), starting with `not json` (an engine \ + writes only its JSON answer to stdout; logs go to stderr); stderr: oops" ); } @@ -560,10 +563,8 @@ mod tests { let engine = FakeEngine::new("rocmd-sigpipe", false, "exit 3"); // Larger than a pipe buffer, so the write reaches the closed read end. let big = "x".repeat(1 << 20); - let envelope = EngineRequestEnvelope { - method: EngineMethod::Healthcheck, - payload: json!({ "blob": big }), - }; + let envelope = + EngineRequestEnvelope::new(EngineMethod::Healthcheck, json!({ "blob": big })); let result = engine_request_via::( engine.path(), &paths, diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index c0579a668c..44b6e2fdc6 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -147,7 +147,7 @@ pub(crate) fn stop_managed_service_via( record.stop_requested_unix_ms = Some(requested_at); } record.write()?; - // Drop the 0600 endpoint key file with the service, rather than stranding a + // Drop the owner-only endpoint key file with the service, rather than stranding a // plaintext secret on disk for something that is no longer running. // Best-effort and idempotent: a missing key file is not an error, so a // loopback service that never had one, and a repeated stop, both pass @@ -159,9 +159,8 @@ pub(crate) fn stop_managed_service_via( // that is otherwise fine, with no way to re-mint it. The marker written // above defers the cleanup instead; see there for who performs it. if all_stopped { - let _ = std::fs::remove_file(rocm_engine_protocol::endpoint_key_file_path( - paths, service_id, - )); + // With any staged copy a store killed before publishing left beside it. + rocm_engine_protocol::remove_endpoint_key_files(paths, service_id); } Ok(json!({ "service": record, @@ -236,12 +235,17 @@ mod tests { fs::create_dir_all(paths.services_dir())?; fs::write(&key_path, "secret-key")?; assert!(key_path.exists()); + // What a `rocm` store killed between staging and publishing leaves: a + // second copy of the key, named by the shared atomic writer. + let staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&key_path, b"secret-key")?; let result = stop_managed_service(&paths, service_id); // Observe the real filesystem state before the blanket temp-dir cleanup, // otherwise remove_dir_all would delete the key file and mask a missing // production cleanup (the regression this test guards). let key_removed = !key_path.exists(); + let staged_removed = !staged.exists(); fs::remove_dir_all(root).ok(); let value = result?; @@ -253,6 +257,7 @@ mod tests { Some("stopped") ); assert!(key_removed, "endpoint key file must be removed after stop"); + assert!(staged_removed, "a staged copy of the key must go with it"); Ok(()) } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index bfb5092ada..9bc53d3b74 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -150,8 +150,14 @@ fn supervise_service_on( let rocm_binary = daemon_host.rocm_binary()?; record.write()?; - let log_file = fs::File::create(&record.log_path) + // Emptied for this run, then reopened for append: a handle with its own + // offset would write over anything appended to the log meanwhile. + fs::File::create(&record.log_path) .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let log_file = fs::OpenOptions::new() + .append(true) + .open(&record.log_path) + .with_context(|| format!("failed to open {}", record.log_path.display()))?; let log_file_err = log_file .try_clone() .context("failed to clone service log file handle")?; @@ -192,6 +198,13 @@ fn supervise_service_on( let mut child = command .spawn() .with_context(|| format!("failed to spawn engine supervisor child for {engine}"))?; + // `Child` itself does not kill on drop. Until the child is waited for, + // leaving this function any other way — an error from a record write below, + // or a panic — terminates and reaps the engine's process tree. What the + // guard does not act on is described on `EngineChild`. + let mut engine_child = EngineChild::new(&mut child); + #[cfg(test)] + tests::engine_spawned(paths, &record.service_id, engine_child.pid()); // Every write from here on goes through `update_supervised_record`: it // applies only this supervisor's own fields to the record as it is on disk @@ -200,11 +213,12 @@ fn supervise_service_on( // // Captured while the child is known alive, so a later stop verifies this // exact process instead of whatever has since inherited its PID. - let engine_pid = child.id(); + let engine_pid = engine_child.pid(); update_supervised_record(paths, &record, |current| { current.record_engine_identity(engine_pid); current.status = "running".to_owned(); - })?; + }) + .context("failed to record the engine's PID")?; let became_ready = wait_for_service_ready( daemon_host, @@ -224,10 +238,13 @@ fn supervise_service_on( current.status = "ready".to_owned(); // The phase only describes the coming-up window; clear it once ready. current.startup_phase = None; - })?; + }) + .context("failed to record the service as ready")?; } - let exit_status = child.wait().context("failed waiting for engine child")?; + let exit_status = engine_child + .wait() + .context("failed waiting for engine child")?; record_supervised_exit(paths, &record, exit_status.success())?; if exit_status.success() { @@ -690,14 +707,13 @@ fn restart_managed_service_via( if let Err(error) = record_spawned_supervisor(paths, record, child.id()) { // The supervisor is alive but unrecorded: it would hold the port while // being invisible to every stop/recovery path. Kill it first. - terminate_recovery_supervisor(&mut child); - let error = error.context("failed to publish the recovering service record"); - return Err(fail_recovery( + return Err(abandon_recovery_supervisor( paths, record, + &mut child, &recovering_snapshot, prior, - error, + error.context("failed to publish the recovering service record"), )); } @@ -708,15 +724,14 @@ fn restart_managed_service_via( let exited = match child.try_wait() { Ok(exited) => exited, Err(error) => { - terminate_recovery_supervisor(&mut child); - let error = anyhow::Error::new(error) - .context("failed to check recovery supervisor startup state"); - return Err(fail_recovery( + return Err(abandon_recovery_supervisor( paths, record, + &mut child, &recorded_snapshot, prior, - error, + anyhow::Error::new(error) + .context("failed to check recovery supervisor startup state"), )); } }; @@ -763,10 +778,224 @@ fn fail_recovery( error } -/// Grace given to a recovery supervisor that must be terminated because its -/// record could not be published. Short: it is milliseconds old. +/// Grace given to a child this process spawned and now has to terminate: a +/// recovery supervisor whose record could not be published, or an engine whose +/// supervisor is leaving before it waited for it. Short: neither has anything +/// to flush that outweighs freeing the port and the GPU. const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); +/// Terminate `child`, which this process spawned and has not reaped, and its +/// descendants, then reap it — with a blocking wait when the termination says +/// the child itself is gone, and otherwise only if it has already exited (see +/// [`terminate_and_reap_with`]). Returns the line describing what may still be +/// running when the termination did not confirm the stop, naming the child as +/// `what`, and `None` when it did. +/// +/// Terminates through [`rocm_core::terminate_recorded_pid_within`], forced and +/// over the child's process tree ([`terminate_child_tree`]). +fn terminate_and_reap( + child: &mut std::process::Child, + identity: &rocm_core::ProcessIdentity, + what: &str, +) -> Option { + terminate_and_reap_with(child, identity, what, terminate_child_tree) +} + +/// The termination [`terminate_and_reap`] runs: forced, over the child's whole +/// process tree, with [`RECOVERY_CLEANUP_GRACE`] for the child itself. +fn terminate_child_tree(identity: &rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome { + rocm_core::terminate_recorded_pid_within(identity, RECOVERY_CLEANUP_GRACE) +} + +/// Write `line`, from [`unconfirmed_termination_line`], to `report`. A write +/// that fails is dropped rather than panicking: this runs from a `Drop`, possibly +/// during an unwind, where a panic aborts the process, and a full disk — which +/// also fails the record writes that lead here — is exactly when it would. +fn report_unconfirmed_termination(report: &mut dyn std::io::Write, line: &str) { + let _ = writeln!(report, "rocmd: {line}"); +} + +/// [`terminate_and_reap`] with the termination supplied by the caller, so a +/// test can hand it outcomes a real child cannot be made to produce. +/// +/// The reap blocks only once the child is known gone. When the termination +/// reports it still alive or unverifiable, a blocking `wait` could hang the +/// caller for good, so the child is reaped only if it has already exited. +fn terminate_and_reap_with( + child: &mut std::process::Child, + identity: &rocm_core::ProcessIdentity, + what: &str, + terminate: impl FnOnce(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, +) -> Option { + let outcome = terminate(identity); + if termination_may_have_left_the_child_running(outcome) { + let _ = child.try_wait(); + } else { + let _ = child.wait(); + } + unconfirmed_termination_line(what, identity.pid, outcome) +} + +/// Whether `outcome` leaves the terminated process itself possibly alive, as +/// against gone with only the processes under it unaccounted for. A +/// [`TimedOut`](rocm_core::TerminationOutcome::TimedOut) can be either: the +/// wait covers the whole tree, so it may be a process under the child that +/// outlived the deadline. +const fn termination_may_have_left_the_child_running( + outcome: rocm_core::TerminationOutcome, +) -> bool { + matches!( + outcome, + rocm_core::TerminationOutcome::TimedOut | rocm_core::TerminationOutcome::Unverified + ) +} + +/// The line reporting a termination that did not confirm the stop, or `None` +/// when it did. Derived from the outcome the reap was decided on, so the two +/// cannot disagree about what may still be running. +fn unconfirmed_termination_line( + what: &str, + pid: u32, + outcome: rocm_core::TerminationOutcome, +) -> Option { + if outcome.stopped() { + return None; + } + let survivor = if termination_may_have_left_the_child_running(outcome) { + format!("the {what} or a process under it") + } else { + format!("a process under the {what}") + }; + Some(format!( + "terminating the {what} (pid {pid}) ended {}: {survivor} may still be running", + outcome.as_str() + )) +} + +// Test-only, thread-scoped count of the engine children `EngineChild` had to +// terminate, so a test can tell a guard that was waited for from one that +// signalled a child which had already exited. +#[cfg(test)] +thread_local! { + static ENGINE_CHILD_TERMINATIONS: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + +/// The engine child of [`supervise_service`], held from the spawn until it is +/// waited for. +/// +/// Dropped before a successful wait — any return between the spawn and the +/// wait, such as a record write or the wait failing, or a panic — the guard +/// terminates and reaps the engine's process tree through +/// [`terminate_and_reap_with`]. A guard whose wait succeeded signals nothing. +/// A termination it cannot confirm is reported as a `rocmd: …` line on +/// `report`. +/// +/// A record write that `update_supervised_record` skips (returning `Ok` +/// without writing) is not a return, so the guard does not act on it. An +/// external kill of the supervisor (a signal, the OOM killer) bypasses the +/// guard: not covered; see ROCMAI-957. +struct EngineChild<'a> { + child: &'a mut std::process::Child, + /// Captured right after the spawn, while the child is certainly alive. + identity: rocm_core::ProcessIdentity, + waited: bool, + /// How the guard terminates the engine: [`terminate_child_tree`] outside + /// tests. + terminate: fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, + /// Where the line about a termination that was not confirmed goes: the + /// supervisor's stderr when `None` + /// (`a_dropped_engine_guard_reports_to_stderr_by_default`). The recovery + /// restart hands the supervisor its service log as stderr + /// (`restart_hands_the_supervisor_its_service_log_as_stderr`). + report: Option<&'a mut dyn std::io::Write>, +} + +impl<'a> EngineChild<'a> { + fn new(child: &'a mut std::process::Child) -> Self { + let identity = rocm_core::ProcessIdentity::capture(child.id()); + Self { + child, + identity, + waited: false, + terminate: terminate_child_tree, + report: None, + } + } + + /// This guard, terminating its engine with `terminate` instead. + #[cfg(test)] + fn terminating_with( + mut self, + terminate: fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, + ) -> Self { + self.terminate = terminate; + self + } + + /// This guard, reporting to `report` instead of stderr. + #[cfg(test)] + fn reporting_to(mut self, report: &'a mut dyn std::io::Write) -> Self { + self.report = Some(report); + self + } + + const fn pid(&self) -> u32 { + self.identity.pid + } + + /// Wait for the engine to exit. + fn wait(&mut self) -> std::io::Result { + let waited = self.child.wait(); + self.settle_wait(waited) + } + + /// Disarm the guard on a wait that returned the engine's exit status, and + /// only then: after a failed one, nothing says the engine is gone, so the + /// guard still terminates it when dropped. Separate so a test can hand it a + /// failed wait. + fn settle_wait( + &mut self, + waited: std::io::Result, + ) -> std::io::Result { + let status = waited?; + self.waited = true; + Ok(status) + } +} + +impl Drop for EngineChild<'_> { + fn drop(&mut self) { + if self.waited { + return; + } + #[cfg(test)] + ENGINE_CHILD_TERMINATIONS.with(|calls| calls.set(calls.get() + 1)); + // This can run during an unwind, where a panic aborts the process. The + // termination does not panic: `terminate_child_tree` and the rocm-core + // routine under it (identity reads, the `/proc` tree walk, signalling, + // the Windows `taskkill` and its wait) use no `unwrap`, `expect` or + // unchecked indexing, and none of their arithmetic can overflow. The reap + // and the report write return errors, which are dropped. It blocks: on + // Linux for about twice `RECOVERY_CLEANUP_GRACE` at most (after `SIGTERM`, + // then after `SIGKILL`), on Windows for the `taskkill` of the tree (bounded + // by `rocm_core::MANAGED_STOP_GRACE`) and then up to one + // `RECOVERY_CLEANUP_GRACE`; the final blocking `wait` runs only once the + // child is known gone, so it returns at once. + // + // The error the supervisor returns next says only why it is leaving, so + // the line below is what records an engine that may have survived. + let unconfirmed = + terminate_and_reap_with(self.child, &self.identity, "engine", self.terminate); + if let Some(line) = unconfirmed { + match self.report.as_deref_mut() { + Some(report) => report_unconfirmed_termination(report, &line), + None => report_unconfirmed_termination(&mut std::io::stderr(), &line), + } + } + } +} + // Test-only, thread-scoped call counter: `restart_managed_service`'s rollback // branch asserts this to prove the kill is actually reached (not just that the // record comes back truthful), since that branch's own supervisor is a @@ -774,29 +1003,60 @@ const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); // whether it is killed, so `process_is_running` on it would not catch the call // being dropped. Thread-local, not a global counter: cargo runs tests // concurrently on separate threads, and this function has another, unrelated -// direct caller (`terminate_recovery_supervisor_kills_and_reaps_the_child`). +// direct caller (`terminate_recovery_supervisor_kills_and_reaps_the_child_and_its_tree`). #[cfg(test)] thread_local! { static TERMINATE_RECOVERY_SUPERVISOR_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// Terminate and reap a recovery supervisor that could not be recorded. +// Test-only, thread-scoped outcome `terminate_recovery_supervisor` reports in +// place of terminating, so a test can drive the restart's error path with an +// outcome a real child cannot be made to produce. +#[cfg(test)] +thread_local! { + static RECOVERY_SUPERVISOR_TERMINATION_STUB: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +/// Terminate a recovery supervisor the restart could not record, with its +/// process tree, through [`terminate_and_reap`], and reap it when it is known +/// gone. After a timed-out or unverified termination it is reaped only if it +/// has already exited, so one that exits later stays unreaped. /// -/// Tree scope, forced: the supervisor may already have started an engine child, -/// and leaving either alive would keep the recorded port occupied by a process -/// no stop or recovery path can find. Reaping afterwards keeps the daemon free -/// of zombies. -fn terminate_recovery_supervisor(child: &mut std::process::Child) { +/// Returns the line describing what may still be running when the termination +/// was not confirmed, for the restart's error ([`abandon_recovery_supervisor`]) +/// rather than for stderr. +fn terminate_recovery_supervisor(child: &mut std::process::Child) -> Option { #[cfg(test)] TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(|calls| calls.set(calls.get() + 1)); - let _outcome = rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::capture(child.id()), - rocm_core::KillScope::Tree, - RECOVERY_CLEANUP_GRACE, - true, - ); - let _ = child.wait(); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + #[cfg(test)] + if let Some(outcome) = RECOVERY_SUPERVISOR_TERMINATION_STUB.with(std::cell::Cell::get) { + return terminate_and_reap_with(child, &identity, "recovery supervisor", |_| outcome); + } + terminate_and_reap(child, &identity, "recovery supervisor") +} + +/// Give up on a recovery supervisor this restart spawned but cannot keep: +/// terminate it, then roll the record back with [`fail_recovery`]. When the +/// termination was not confirmed, `error` carries the line saying what may still +/// be running. Every such exit from `restart_managed_service_via` goes through +/// here, so none of them can terminate without reporting, or report without +/// terminating. +fn abandon_recovery_supervisor( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + child: &mut std::process::Child, + snapshot: &ManagedServiceRecord, + prior: ManagedServiceRecord, + error: anyhow::Error, +) -> anyhow::Error { + let error = match terminate_recovery_supervisor(child) { + Some(unconfirmed) => error.context(unconfirmed), + None => error, + }; + fail_recovery(paths, record, snapshot, prior, error) } /// Between the restart's first write — which clears the recorded PIDs, since @@ -1728,6 +1988,63 @@ mod tests { Ok(()) } + /// A line the supervisor launched by the recovery restart writes to stderr + /// turns up in the record's log: the stand-in `rocmd` writes one, and the + /// test reads it back from `record.log_path`. + #[cfg(target_os = "linux")] + #[test] + fn restart_hands_the_supervisor_its_service_log_as_stderr() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + assert!( + find_on_path("setsid").is_some(), + "this test needs `setsid` on PATH: the restart runs it" + ); + let (root, paths) = temp_app_paths("restart-supervisor-stderr-log"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let seen = root.join("rocmd-ran"); + let host = DaemonHost::new(root.join("rocm"), true); + write_executable_script( + &root.join("rocmd"), + &tools, + &format!( + "echo supervisor-stderr-reaches-the-log >&2\n: > '{}'\nexec '{}' 3\n", + seen.display(), + tools.sleep.display() + ), + )?; + + let service_id = "svc-restart-supervisor-stderr-log"; + let mut record = identity_probe_record(&paths, service_id, 11476); + record.status = "failed".to_owned(); + record.write()?; + + // As in `restart_runs_supervise_on_the_rocmd_it_is_given`: retried for + // an exec that loses a race with an unrelated fork. + let mut outcome = Err(anyhow::anyhow!("not attempted")); + for _ in 0..10 { + outcome = restart_managed_service_on(&paths, &mut record, &host); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !seen.exists() && std::time::Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); + } + if seen.exists() { + break; + } + } + let log = fs::read_to_string(&record.log_path).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + + outcome?; + assert!( + log.contains("supervisor-stderr-reaches-the-log\n"), + "the supervisor's stderr must be the service log at {}: {log:?}", + record.log_path.display() + ); + Ok(()) + } + /// With no `rocmd` to run, the restart reports it, and the record is exactly /// as the restart found it: it is not marked `recovering` for a supervisor /// that was never started. @@ -2408,130 +2725,1229 @@ mod tests { Ok(()) } + /// The environment a [`recovery_cleanup_sleeper_child`] reads: the marker it + /// writes once it is sleeping, how long it sleeps, and, when set, the marker + /// of a grandchild sleeper it starts first. + const SLEEPER_MARKER_ENV: &str = "ROCMD_TEST_SLEEPER_CHILD"; + const SLEEPER_SECS_ENV: &str = "ROCMD_TEST_SLEEPER_SECS"; + const SLEEPER_GRANDCHILD_ENV: &str = "ROCMD_TEST_SLEEPER_GRANDCHILD"; + + /// A test that sleeps when spawned by [`spawn_sleeper`] and does nothing + /// otherwise. + /// + /// It writes its marker file only once it is sleeping, and the spawner waits + /// for that marker: a filter that no longer names this test runs zero tests + /// and exits at once, which a kill test would otherwise read as a kill. That + /// happened once already, when this test moved modules and the hard-coded + /// filter kept the old path. + /// + /// Asked for a grandchild, it first starts another copy of itself and waits + /// for that one to sleep, then writes the grandchild's PID into its own + /// marker: a process tree two deep, the shape of a supervisor over an + /// engine, on every platform the suite runs on. #[test] - #[ignore = "spawned as a child process by terminate_recovery_supervisor_kills_and_reaps_the_child"] + #[ignore = "spawned as a child process by spawn_sleeper"] fn recovery_cleanup_sleeper_child() { - if std::env::var_os("ROCMD_TEST_SLEEPER_CHILD").is_none() { + let Some(marker) = std::env::var_os(SLEEPER_MARKER_ENV) else { return; + }; + let secs = std::env::var(SLEEPER_SECS_ENV) + .ok() + .and_then(|secs| secs.parse().ok()) + .unwrap_or(120); + let grandchild = std::env::var_os(SLEEPER_GRANDCHILD_ENV).map(|grandchild_marker| { + let grandchild = sleeper_command(Path::new(&grandchild_marker), secs) + .spawn() + .expect("spawn the grandchild sleeper"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while !Path::new(&grandchild_marker).exists() && std::time::Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); + } + grandchild + }); + let contents = grandchild + .as_ref() + .map(|grandchild| grandchild.id().to_string()) + .unwrap_or_default(); + // Written beside the marker and renamed onto it, so the spawner never + // reads a marker that exists but is not yet written. + let marker = Path::new(&marker); + let partial = marker.with_extension("partial"); + fs::write(&partial, contents).expect("write the sleeper's marker"); + fs::rename(&partial, marker).expect("publish the sleeper's marker"); + thread::sleep(Duration::from_secs(secs)); + if let Some(mut grandchild) = grandchild { + let _ = grandchild.wait(); } - thread::sleep(Duration::from_mins(2)); } - #[test] - fn terminate_recovery_supervisor_kills_and_reaps_the_child() { - // A real long-lived process stands in for a supervisor that was spawned - // but whose record could not be published. - let mut child = ProcessCommand::new(std::env::current_exe().expect("test binary")) - .arg("watchers::tests::recovery_cleanup_sleeper_child") + /// The libtest filter naming [`recovery_cleanup_sleeper_child`], derived from + /// this module's own path so it cannot drift from the test's location again. + /// libtest names tests without the crate, so the crate segment is dropped. + fn sleeper_child_test_name() -> String { + let module = module_path!() + .split_once("::") + .map_or(module_path!(), |(_crate, module)| module); + format!("{module}::recovery_cleanup_sleeper_child") + } + + /// This test binary, set up to run [`recovery_cleanup_sleeper_child`] for + /// `secs` seconds, writing `marker` once it sleeps. + fn sleeper_command(marker: &Path, secs: u64) -> ProcessCommand { + let mut command = ProcessCommand::new(std::env::current_exe().expect("test binary")); + command + .arg(sleeper_child_test_name()) .arg("--exact") .arg("--ignored") - .env("ROCMD_TEST_SLEEPER_CHILD", "1") + .env(SLEEPER_MARKER_ENV, marker) + .env(SLEEPER_SECS_ENV, secs.to_string()) + .env_remove(SLEEPER_GRANDCHILD_ENV) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn sleeper child"); + .stderr(Stdio::null()); + command + } + + /// Spawn [`recovery_cleanup_sleeper_child`] for `secs` seconds, optionally + /// over a grandchild, and return once it is provably sleeping — so that it + /// ending quickly can only be the work of a kill — with the grandchild's PID. + fn spawn_sleeper( + label: &str, + secs: u64, + with_grandchild: bool, + ) -> (std::process::Child, Option) { + let root = unique_test_root(&format!("rocmd-sleeper-{label}-{}", std::process::id())); + let marker = root.join("sleeping"); + let grandchild_marker = root.join("grandchild-sleeping"); + fs::remove_file(&marker).ok(); + fs::remove_file(&grandchild_marker).ok(); + let mut command = sleeper_command(&marker, secs); + if with_grandchild { + command.env(SLEEPER_GRANDCHILD_ENV, &grandchild_marker); + } + let mut child = command.spawn().expect("spawn sleeper child"); + let deadline = std::time::Instant::now() + Duration::from_mins(1); + // A marker asked to name a grandchild counts only once it does: the + // sleeper renames it into place, and this holds even if it did not. + let contents = loop { + if let Ok(contents) = fs::read_to_string(&marker) + && (!with_grandchild || contents.trim().parse::().is_ok()) + { + break contents; + } + if let Ok(Some(status)) = child.try_wait() { + fs::remove_dir_all(&root).ok(); + panic!( + "the sleeper exited ({status}) before it started sleeping: the filter {:?} \ + does not name it", + sleeper_child_test_name() + ); + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + let _ = child.wait(); + fs::remove_dir_all(&root).ok(); + panic!("the sleeper did not start sleeping within 60s"); + } + thread::sleep(Duration::from_millis(20)); + }; + fs::remove_dir_all(&root).ok(); + let grandchild = with_grandchild.then(|| { + contents + .trim() + .parse() + .expect("the loop above waited for a parsable grandchild PID") + }); + (child, grandchild) + } + + /// Spawn a two-minute sleeper with a grandchild under it. + fn spawn_sleeper_tree(label: &str) -> (std::process::Child, u32) { + let (child, grandchild) = spawn_sleeper(label, 120, true); + (child, grandchild.expect("asked for a grandchild")) + } + + /// Whether `pid` stops running within a few seconds. A grandchild killed + /// with its parent is reparented and reaped by someone else, so it can stay + /// visible as a zombie for a moment after the kill returns. + fn gone_soon(pid: u32) -> bool { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while rocm_core::process_is_running(pid) { + if std::time::Instant::now() > deadline { + return false; + } + thread::sleep(Duration::from_millis(20)); + } + true + } + + /// Kill `pid` if it is still running, so a failing test leaves no sleeper. + fn kill_if_running(pid: u32) { + if rocm_core::process_is_running(pid) { + let _ = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(pid), + rocm_core::KillScope::Single, + Duration::from_secs(1), + true, + ); + } + } + + #[test] + fn terminate_recovery_supervisor_kills_and_reaps_the_child_and_its_tree() { + // A real long-lived process tree stands in for a supervisor that was + // spawned, and had started its engine, but whose record could not be + // published. + let (mut child, grandchild) = spawn_sleeper_tree("recovery-supervisor"); let child_pid = child.id(); let started = std::time::Instant::now(); - terminate_recovery_supervisor(&mut child); + let unconfirmed = terminate_recovery_supervisor(&mut child); + let elapsed = started.elapsed(); + let grandchild_gone = gone_soon(grandchild); + kill_if_running(grandchild); + // The sleeper sleeps for two minutes by itself: only a kill returns // well before that, so waiting it out must not pass. assert!( - started.elapsed() < Duration::from_secs(30), - "terminate_recovery_supervisor took {:?}: the child was waited out, not killed", - started.elapsed() + elapsed < Duration::from_secs(30), + "terminate_recovery_supervisor took {elapsed:?}: the child was waited out, not killed" ); - // The helper reaps, so the pid must be gone rather than a zombie. assert!( !rocm_core::process_is_running(child_pid), "an unrecordable supervisor must be terminated and reaped" ); + assert!( + grandchild_gone, + "the process under the supervisor (pid {grandchild}) must be terminated with it" + ); + assert_eq!( + unconfirmed, None, + "a confirmed termination leaves nothing for the restart's error" + ); // The helper already reaped it; wait again so the child is provably // handled on every path in this function (and reaping is idempotent). assert!(child.wait().is_ok()); } - /// If publishing the `recovering` transition fails, nothing has been - /// spawned yet: the rollback at that first write must undo the in-memory - /// transition and leave the on-disk record exactly as it was, not a - /// half-applied `recovering` row. The manifest write goes through - /// [`write_file_atomically`][rocm_core_atomic_write]-style write-then-rename, - /// so a read-only *file* no longer blocks it (rename ignores the - /// destination's own permission bits) — a read-only *directory* still does, - /// since it can't hold the new temp file the rename needs. - /// - /// [rocm_core_atomic_write]: rocm_core::atomic_write::write_file_atomically - /// - /// Unix only and skipped as root, where a read-only mode is not enforced - /// and the write would simply succeed. - #[cfg(unix)] + /// A supervisor that leaves before waiting for its engine — an error between + /// the spawn and the wait, or a panic — drops the guard, and the engine must + /// go with it: terminated with the processes under it, and reaped rather + /// than left a zombie. #[test] - fn restart_rolls_back_the_in_memory_record_when_the_first_write_fails() -> Result<()> { - use std::os::unix::fs::PermissionsExt; + fn an_engine_child_dropped_before_its_wait_is_terminated_and_reaped() { + let (mut child, grandchild) = spawn_sleeper_tree("engine-child-drop"); + let child_pid = child.id(); + let terminations_before = ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get); - if rocm_core::openmpi::running_as_root() { - return Ok(()); + let started = std::time::Instant::now(); + drop(EngineChild::new(&mut child)); + let elapsed = started.elapsed(); + let still_running = rocm_core::process_is_running(child_pid); + if still_running { + let _ = child.kill(); } + let _ = child.wait(); + let grandchild_gone = gone_soon(grandchild); + kill_if_running(grandchild); - let (root, paths) = temp_app_paths("restart-rollback-first-write"); - paths.ensure()?; - let service_id = "svc-restart-rollback-first-write"; - let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); - record.status = "failed".to_owned(); - record.write()?; - - let services_dir = paths.services_dir(); - // The launch lock and service log files are both created lazily, on - // first acquire and on first open respectively; pre-create them so - // making the directory read-only below blocks only the manifest's - // write-then-rename, not those earlier opens. - fs::File::create(paths.managed_launch_lock_path())?; - fs::File::create(&record.log_path)?; - fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; + assert_eq!( + ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get), + terminations_before + 1, + "an engine that was never waited for must be terminated when its guard drops" + ); + // The sleeper sleeps for two minutes by itself. + assert!( + elapsed < Duration::from_secs(30), + "dropping the guard took {elapsed:?}: the engine was waited out, not killed" + ); + assert!( + !still_running, + "the engine must be terminated and reaped when its guard drops unwaited" + ); + assert!( + grandchild_gone, + "the process under the engine (pid {grandchild}) must be terminated with it" + ); + } - let result = restart_managed_service(&paths, &mut record); + /// The other side: an engine the supervisor waited for has exited and been + /// reaped, so dropping its guard signals nothing — its PID may already belong + /// to someone else. + #[test] + fn an_engine_child_that_was_waited_for_is_not_signalled_on_drop() { + // The test binary with a filter matching no test exits at once, cleanly. + let mut child = ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("--exact") + .arg("no-test-has-this-name") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a child that exits at once"); + let terminations_before = ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get); - // Restore write access so the rest of the write (if any landed) can be - // read back, and so cleanup can remove the directory. - fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); + let mut engine = EngineChild::new(&mut child); + let status = engine.wait().expect("wait for the child"); + drop(engine); - let error = result.expect_err("a read-only manifest must fail the first write"); - assert!( - format!("{error:#}").contains("failed to publish the recovering service record"), - "{error:#}" - ); + assert!(status.success(), "the child exits cleanly: {status}"); assert_eq!( - record.status, "failed", - "a failed first write must restore the in-memory record rather than leave it \ - \"recovering\"" + ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get), + terminations_before, + "a waited-for engine must not be signalled when its guard drops" ); + } + + /// A failed wait does not disarm the guard: nothing says the engine is gone + /// after one, so dropping the guard still terminates it and what is under it. + #[test] + fn a_failed_wait_leaves_the_engine_guard_armed() { + let (mut child, grandchild) = spawn_sleeper_tree("engine-child-failed-wait"); + let child_pid = child.id(); + let terminations_before = ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get); + + let mut engine = EngineChild::new(&mut child); + let settled = engine.settle_wait(Err(std::io::Error::other("injected wait failure"))); + drop(engine); + let still_running = rocm_core::process_is_running(child_pid); + if still_running { + let _ = child.kill(); + } + let _ = child.wait(); + let grandchild_gone = gone_soon(grandchild); + kill_if_running(grandchild); + + assert!(settled.is_err(), "the failed wait is handed back"); assert_eq!( - persisted?.status, "failed", - "nothing must reach disk when the first write itself fails" + ENGINE_CHILD_TERMINATIONS.with(std::cell::Cell::get), + terminations_before + 1, + "a guard whose wait failed must still terminate the engine when dropped" + ); + assert!( + !still_running, + "the engine must be terminated and reaped after a failed wait" + ); + assert!( + grandchild_gone, + "the process under the engine (pid {grandchild}) must be terminated too" ); - Ok(()) } - /// The first-write rollback above covers a failure before anything is - /// spawned; this one covers the second write, after the supervisor exists. - /// The spawn seam corrupts the manifest's content right after the first - /// write (`recovering`, no supervisor yet), so - /// `record_spawned_supervisor`'s read fails. That must kill the spawned - /// supervisor and restore the prior record — and the rollback write itself - /// must still land, since corrupting *content* leaves the file a writable - /// regular file. + /// A kill of `pid` that fires after a delay unless it is cancelled first. /// - /// The spawned supervisor here is a re-exec of this very test binary, - /// which rejects its `supervise` arguments and exits on its own almost - /// immediately regardless of whether anything kills it, so asserting + /// The reap tests use it against a sleeper that would otherwise sleep for two + /// minutes, so whether a call blocked on the child is decided by whether the + /// kill had to fire before the call returned — not by racing a short + /// sleeper's own exit against the host's scheduling. + struct DelayedKill { + cancel: std::sync::mpsc::Sender<()>, + handle: thread::JoinHandle, + } + + impl DelayedKill { + fn start(pid: u32, after: Duration) -> Self { + let identity = rocm_core::ProcessIdentity::capture(pid); + let (cancel, cancelled) = std::sync::mpsc::channel::<()>(); + let handle = thread::spawn(move || { + if matches!( + cancelled.recv_timeout(after), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ) { + let _ = rocm_core::terminate_verified( + &identity, + rocm_core::KillScope::Single, + Duration::from_secs(1), + true, + ); + true + } else { + false + } + }); + Self { cancel, handle } + } + + /// Cancel the kill if it has not fired yet, and say whether it had. + fn finish(self) -> bool { + let _ = self.cancel.send(()); + self.handle.join().unwrap_or(false) + } + } + + /// A termination that leaves the child possibly alive — the child or a + /// process under it outlived the deadline, or its identity could not be + /// confirmed — must not be followed by a blocking reap, which would hang the + /// caller on a process that may never exit. The child is + /// left unreaped, and the line says what may still be running and why. + /// + /// A kill fires after 20 s so that a blocking reap fails the test then + /// rather than after the sleeper's two minutes. On Linux the decisive check + /// does not depend on timing at all: a child the call did not reap keeps its + /// PID, as a zombie if the kill has fired, while a blocking reap releases + /// it. Windows has no zombie to look at, so there the check is that the + /// call returned before the kill had to fire. + #[test] + fn a_termination_that_leaves_the_child_alive_neither_blocks_nor_stays_silent() { + for outcome in [ + rocm_core::TerminationOutcome::TimedOut, + rocm_core::TerminationOutcome::Unverified, + ] { + let (mut child, _) = spawn_sleeper(outcome.as_str(), 120, false); + let pid = child.id(); + let identity = rocm_core::ProcessIdentity::capture(pid); + let kill = DelayedKill::start(pid, Duration::from_secs(20)); + + let line = terminate_and_reap_with(&mut child, &identity, "engine", |_| outcome); + let pid_still_there = rocm_core::process_is_running(pid); + let kill_fired = kill.finish(); + let _ = child.kill(); + let _ = child.wait(); + + #[cfg(not(windows))] + assert!( + pid_still_there, + "{outcome:?}: the call reaped a child the termination did not stop, so it \ + blocked on it (the kill fired: {kill_fired})" + ); + #[cfg(windows)] + assert!( + !kill_fired, + "{outcome:?}: the call returned only after the child was killed, so it \ + blocked on it (PID still there: {pid_still_there})" + ); + assert_eq!( + line, + Some(format!( + "terminating the engine (pid {pid}) ended {}: the engine or a process \ + under it may still be running", + outcome.as_str() + )), + "{outcome:?}" + ); + } + } + + /// After an outcome that leaves the child possibly alive, a child that has in + /// fact already exited is still reaped, not left a zombie. The child is + /// waited for with `WNOWAIT` first, which sees the exit without reaping it, + /// so it is certainly an unreaped exited child when the call is made. A + /// zombie still answers `kill(pid, 0)`, so the PID being gone afterwards + /// means the call reaped it. + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] // libc FFI: waitid + fn an_unconfirmed_termination_still_reaps_a_child_that_already_exited() { + for outcome in [ + rocm_core::TerminationOutcome::TimedOut, + rocm_core::TerminationOutcome::Unverified, + ] { + // The test binary with a filter matching no test exits at once. + let mut child = ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("--exact") + .arg("no-test-has-this-name") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a child that exits at once"); + let pid = child.id(); + let identity = rocm_core::ProcessIdentity::capture(pid); + let id = libc::id_t::from(pid); + // SAFETY: `info` is a valid, zeroed `siginfo_t` for `waitid` to fill. + let waited = unsafe { + let mut info: libc::siginfo_t = std::mem::zeroed(); + libc::waitid( + libc::P_PID, + id, + &raw mut info, + libc::WEXITED | libc::WNOWAIT, + ) + }; + assert_eq!( + waited, 0, + "{outcome:?}: wait for the child's exit without reaping it" + ); + assert!( + rocm_core::process_is_running(pid), + "{outcome:?}: precondition: the exited child is an unreaped zombie" + ); + + let line = terminate_and_reap_with(&mut child, &identity, "engine", |_| outcome); + let reaped = !rocm_core::process_is_running(pid); + let _ = child.wait(); + + assert!( + reaped, + "{outcome:?}: a child that had already exited must be reaped, not left a zombie" + ); + assert!( + line.is_some(), + "{outcome:?}: the unconfirmed outcome is still reported" + ); + } + } + + /// Run `terminate_and_reap_with` against a two-minute sleeper, with a + /// termination that reports `outcome` without touching it, while a kill + /// fires after 5 s. Returns whether that kill had fired by the time the call + /// returned, whether the sleeper's PID is still there afterwards (as a + /// process or an unreaped zombie), and the line. + /// + /// A reap that blocks until the child exits cannot return before the kill, + /// however the host schedules the threads; one that only looks once returns + /// at once with the child alive. + fn reap_after_a_reported_stop( + outcome: rocm_core::TerminationOutcome, + what: &str, + ) -> (bool, u32, bool, Option) { + let (mut child, _) = spawn_sleeper(outcome.as_str(), 120, false); + let pid = child.id(); + let identity = rocm_core::ProcessIdentity::capture(pid); + let kill = DelayedKill::start(pid, Duration::from_secs(5)); + + let line = terminate_and_reap_with(&mut child, &identity, what, |_| outcome); + let kill_fired = kill.finish(); + let still_there = rocm_core::process_is_running(pid); + if still_there { + let _ = child.kill(); + let _ = child.wait(); + } + (kill_fired, pid, still_there, line) + } + + /// A termination that reports the child gone is followed by a reap that + /// waits for it, so it does not linger as a zombie, and a confirmed stop + /// produces no line. + #[test] + fn a_termination_that_stops_the_child_reaps_it_and_reports_nothing() { + let (kill_fired, _, still_there, line) = + reap_after_a_reported_stop(rocm_core::TerminationOutcome::Forced, "engine"); + + assert!( + kill_fired, + "the reap returned while the child was still running: it did not wait" + ); + assert!( + !still_there, + "a child the termination reports stopped must be waited for and reaped" + ); + assert_eq!(line, None, "a confirmed stop reports nothing"); + } + + /// A termination that saw the child exit but could not account for the + /// processes under it: the child is waited for and reaped, and the line + /// names what may still be running — a process under it, not the child. + #[test] + fn a_termination_with_unconfirmed_descendants_reaps_the_child_and_says_so() { + let (kill_fired, pid, still_there, line) = reap_after_a_reported_stop( + rocm_core::TerminationOutcome::DescendantsUnconfirmed, + "recovery supervisor", + ); + + assert!( + kill_fired, + "the reap returned while the child was still running: it did not wait" + ); + assert!(!still_there, "the child must be waited for and reaped"); + assert_eq!( + line, + Some(format!( + "terminating the recovery supervisor (pid {pid}) ended descendants_unconfirmed: \ + a process under the recovery supervisor may still be running" + )) + ); + } + + /// An engine guard dropped unwaited whose termination is not confirmed writes + /// the line saying what may still be running to its report, and leaves the + /// engine unreaped rather than blocking on it. + #[test] + fn a_dropped_engine_guard_logs_a_termination_it_could_not_confirm() { + let (mut child, _) = spawn_sleeper("engine-unconfirmed", 120, false); + let pid = child.id(); + let mut log = Vec::new(); + // As in `a_termination_that_leaves_the_child_alive_...`: a blocking reap + // fails after 20 s instead of two minutes, and on Linux the check is the + // unreaped PID, whatever the timing. + let kill = DelayedKill::start(pid, Duration::from_secs(20)); + + drop( + EngineChild::new(&mut child) + .terminating_with(|_| rocm_core::TerminationOutcome::TimedOut) + .reporting_to(&mut log), + ); + let pid_still_there = rocm_core::process_is_running(pid); + let kill_fired = kill.finish(); + let _ = child.kill(); + let _ = child.wait(); + + #[cfg(not(windows))] + assert!( + pid_still_there, + "the guard reaped an engine its termination did not stop, so it blocked on it \ + (the kill fired: {kill_fired})" + ); + #[cfg(windows)] + assert!( + !kill_fired, + "the guard returned only after the engine was killed, so it blocked on it (PID \ + still there: {pid_still_there})" + ); + assert_eq!( + String::from_utf8(log).expect("the log is text"), + format!( + "rocmd: terminating the engine (pid {pid}) ended timed_out: the engine or a \ + process under it may still be running\n" + ) + ); + } + + /// Run by [`a_dropped_engine_guard_reports_to_stderr_by_default`] in a child + /// process whose stderr it captures: drops an unwaited guard that reports + /// nowhere in particular, over a sleeper its stub termination leaves alive, + /// and prints the sleeper's PID so the parent can check the line names it. + #[test] + #[ignore = "spawned as a child process by a_dropped_engine_guard_reports_to_stderr_by_default"] + fn engine_guard_default_report_child() { + if std::env::var_os(GUARD_REPORT_CHILD_ENV).is_none() { + return; + } + let (mut child, _) = spawn_sleeper("engine-default-report", 120, false); + println!("engine-pid={}", child.id()); + drop( + EngineChild::new(&mut child) + .terminating_with(|_| rocm_core::TerminationOutcome::TimedOut), + ); + let _ = child.kill(); + let _ = child.wait(); + } + + const GUARD_REPORT_CHILD_ENV: &str = "ROCMD_TEST_GUARD_REPORT_CHILD"; + + /// A guard given no report writes to the process's own stderr, the + /// production default. Driven in a child process so its real stderr can be + /// read back. + #[test] + fn a_dropped_engine_guard_reports_to_stderr_by_default() { + let root = unique_test_root(&format!( + "rocmd-guard-default-report-{}", + std::process::id() + )); + let (stdout_path, stderr_path) = (root.join("stdout"), root.join("stderr")); + let mut run = ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg({ + let module = module_path!() + .split_once("::") + .map_or(module_path!(), |(_crate, module)| module); + format!("{module}::engine_guard_default_report_child") + }) + .arg("--exact") + .arg("--ignored") + .arg("--nocapture") + .env(GUARD_REPORT_CHILD_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::from( + fs::File::create(&stdout_path).expect("stdout file"), + )) + .stderr(Stdio::from( + fs::File::create(&stderr_path).expect("stderr file"), + )) + .spawn() + .expect("spawn the child test"); + let deadline = std::time::Instant::now() + Duration::from_mins(1); + let status = loop { + if let Ok(Some(status)) = run.try_wait() { + break Some(status); + } + if std::time::Instant::now() > deadline { + let _ = run.kill(); + let _ = run.wait(); + break None; + } + thread::sleep(Duration::from_millis(20)); + }; + let stdout = fs::read_to_string(&stdout_path).unwrap_or_default(); + let stderr = fs::read_to_string(&stderr_path).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + // Searched for, not matched at a line start: with `--nocapture` libtest + // may print the test's name on the same line just before it. + let pid: Option = stdout + .split_once("engine-pid=") + .map(|(_, rest)| rest.chars().take_while(char::is_ascii_digit).collect()) + .filter(|pid: &String| !pid.is_empty()); + if status.is_none() { + // Killing the child test left its sleeper behind. + if let Some(sleeper) = pid.as_deref().and_then(|pid| pid.parse().ok()) { + kill_if_running(sleeper); + } + } + + assert!( + status.is_some_and(|status| status.success()), + "the child test did not finish cleanly ({status:?}): {stdout}{stderr}" + ); + let pid = pid.unwrap_or_else(|| panic!("the child test ran no guard: {stdout}")); + assert!( + stderr.contains(&format!( + "rocmd: terminating the engine (pid {pid}) ended timed_out: the engine or a \ + process under it may still be running\n" + )), + "the default report must reach stderr: {stderr:?}" + ); + } + + /// A report that cannot be written — a full disk fails the log the same way + /// it fails the record writes that lead here — is dropped, not a panic: the + /// guard runs from `Drop`, possibly during an unwind, where a panic aborts + /// the process. + #[test] + fn a_report_that_cannot_be_written_does_not_panic() { + struct Full; + impl std::io::Write for Full { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("no space left on device")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (mut child, _) = spawn_sleeper("engine-unconfirmed-full-log", 120, false); + let mut full = Full; + // Bounds the test if a regression makes the guard block on the engine. + let kill = DelayedKill::start(child.id(), Duration::from_secs(20)); + + let dropped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drop( + EngineChild::new(&mut child) + .terminating_with(|_| rocm_core::TerminationOutcome::TimedOut) + .reporting_to(&mut full), + ); + })); + kill.finish(); + let _ = child.kill(); + let _ = child.wait(); + + assert!(dropped.is_ok(), "a failed write to the log must not panic"); + } + + /// What [`engine_spawned`] does for one registered service, and what it saw. + /// Only the Linux tests register one; elsewhere the seam is compiled but + /// never armed. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + struct EngineSpawnSeam { + service_id: String, + /// Overwrite the service's manifest with something that does not parse, + /// so the supervisor's next record write fails. + corrupt_manifest: bool, + /// A file the stand-in engine writes its own child's PID to, if it + /// starts one. The seam waits for it before going on, so the engine is + /// a process tree by the time anything terminates it. + grandchild_pidfile: Option, + /// The engine PID the supervisor reported, once it spawned one. + engine_pid: Option, + /// The PID of the process under the engine, read from the pidfile. + grandchild_pid: Option, + } + + /// Services [`engine_spawned`] acts for. Keyed by service id, so a test + /// running in parallel with an unregistered service is unaffected. + static ENGINE_SPAWN_SEAMS: std::sync::Mutex> = + std::sync::Mutex::new(Vec::new()); + + /// Test seam run by `supervise_service_on` right after the engine spawn, before + /// the write that records the engine's PID. For a service registered in + /// [`ENGINE_SPAWN_SEAMS`] it remembers the engine PID — which the record may + /// never name — waits for the engine to start a child of its own and + /// remembers that one too, and can corrupt the manifest so the write fails. + /// A corrupt manifest fails the write as root too, which a read-only + /// directory does not. + pub(super) fn engine_spawned(paths: &AppPaths, service_id: &str, engine_pid: u32) { + let pidfile = { + let mut seams = ENGINE_SPAWN_SEAMS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(seam) = seams.iter_mut().find(|seam| seam.service_id == service_id) else { + return; + }; + seam.engine_pid = Some(engine_pid); + seam.grandchild_pidfile.clone() + }; + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let grandchild_pid = pidfile.and_then(|pidfile| { + loop { + let pid = fs::read_to_string(&pidfile) + .ok() + .and_then(|text| text.trim().parse::().ok()); + if pid.is_some() || std::time::Instant::now() > deadline { + break pid; + } + thread::sleep(Duration::from_millis(20)); + } + }); + let mut seams = ENGINE_SPAWN_SEAMS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(seam) = seams.iter_mut().find(|seam| seam.service_id == service_id) { + seam.grandchild_pid = grandchild_pid; + if seam.corrupt_manifest { + fs::write( + paths.service_manifest_path(service_id), + b"not a service record", + ) + .expect("corrupt the service manifest"); + } + } + } + + /// Register `service_id` with [`engine_spawned`]; read what it saw back with + /// [`take_engine_spawn_seam`]. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn register_engine_spawn_seam( + service_id: &str, + corrupt_manifest: bool, + grandchild_pidfile: Option, + ) { + ENGINE_SPAWN_SEAMS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(EngineSpawnSeam { + service_id: service_id.to_owned(), + corrupt_manifest, + grandchild_pidfile, + engine_pid: None, + grandchild_pid: None, + }); + } + + /// Unregister `service_id`, returning the engine PID and the PID of the + /// process under it that [`engine_spawned`] saw. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn take_engine_spawn_seam(service_id: &str) -> (Option, Option) { + let mut seams = ENGINE_SPAWN_SEAMS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + seams + .iter() + .position(|seam| seam.service_id == service_id) + .map_or((None, None), |index| { + let seam = seams.remove(index); + (seam.engine_pid, seam.grandchild_pid) + }) + } + + /// The body of a stand-in `rocm` whose engine is a shell running a two-minute + /// `sleep` under it, writing the sleep's PID to `pidfile`: a process tree, as + /// a real engine is. `stdio` is the `__engine-stdio` branch's body. + #[cfg(target_os = "linux")] + fn stand_in_rocm_with_an_engine_tree( + tools: &crate::test_support::ScriptTools, + pidfile: &Path, + stdio: &str, + ) -> String { + format!( + "case \"$1\" in\n\ + __engine-serve-http)\n '{sleep}' 120 &\n echo $! > '{pidfile}'\n wait\n ;;\n\ + __engine-stdio)\n{stdio}\n ;;\n\ + *) exit 2 ;;\nesac\n", + sleep = tools.sleep.display(), + pidfile = pidfile.display(), + ) + } + + /// Run `supervise_service_on` with the `rocm` at `root/rocm` beside a + /// `rocmd` that is not there. Bounded: if the supervisor does not return + /// within a minute, the engine and the process under it, as recorded by the + /// engine-spawn seam the caller registered, are killed before the test fails. + #[cfg(target_os = "linux")] + fn supervise_on_stand_in_rocm( + root: &Path, + paths: &AppPaths, + service_id: &'static str, + port: u16, + ) -> std::result::Result<(), String> { + let mut outcome = Err("not attempted".to_owned()); + for _ in 0..10 { + let (sender, receiver) = std::sync::mpsc::channel(); + let (thread_paths, daemon_host) = + (paths.clone(), DaemonHost::new(root.join("rocmd"), false)); + thread::spawn(move || { + let outcome = supervise_service_on( + &daemon_host, + &thread_paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + port, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + let received = receiver.recv_timeout(Duration::from_mins(1)); + outcome = if let Ok(outcome) = received { + outcome + } else { + let panicked = matches!( + received, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) + ); + // Unparseable, so the supervisor's exit write fails and it + // returns instead of exiting the test binary once the engine + // it is waiting on dies. + fs::write( + paths.service_manifest_path(service_id), + b"not a service record", + ) + .ok(); + let (engine, grandchild) = take_engine_spawn_seam(service_id); + engine + .into_iter() + .chain(grandchild) + .for_each(kill_if_running); + assert!( + !panicked, + "the supervisor thread panicked (its message is above); its engine (from \ + the registered seam) is killed before the test fails" + ); + panic!( + "the supervisor did not return within a minute; its engine (from the \ + registered seam) is killed before the test fails" + ) + }; + // The first exec of a new file can lose a race with an unrelated + // fork (ETXTBSY): the spawn fails at once, and the next attempt + // starts again. + if !matches!(&outcome, Err(error) if error.contains("Text file busy")) { + break; + } + } + outcome + } + + /// Assert that the supervisor failed at a record write and took its whole + /// engine tree with it, killing whatever survived first. + #[cfg(target_os = "linux")] + fn assert_engine_tree_terminated( + outcome: std::result::Result<(), String>, + service_id: &str, + failed_write: &str, + engine: Option, + grandchild: Option, + ) { + let engine_survived = engine.is_some_and(rocm_core::process_is_running); + let grandchild_survived = grandchild.is_some_and(|pid| !gone_soon(pid)); + engine + .into_iter() + .chain(grandchild) + .for_each(kill_if_running); + + let error = outcome.expect_err("a record write after the spawn was made to fail"); + assert!( + error.contains(failed_write) && error.contains(service_id), + "failed somewhere other than the {failed_write:?} write: {error}" + ); + assert!(engine.is_some(), "the engine was never spawned: {error}"); + assert!( + grandchild.is_some(), + "the engine never started the process under it: {error}" + ); + assert!( + !engine_survived, + "the supervisor returned with its engine {engine:?} still running" + ); + assert!( + !grandchild_survived, + "the supervisor returned with the process under its engine {grandchild:?} still \ + running" + ); + } + + /// The write that records the engine's PID fails after the engine started. + /// The supervisor returns the error — and must take the engine, and what the + /// engine started, with it, although no record names the engine's PID. + #[cfg(target_os = "linux")] + #[test] + fn supervise_terminates_the_engine_when_recording_its_pid_fails() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("supervise-engine-pid-write-fails"); + paths.ensure()?; + let pidfile = root.join("engine-child.pid"); + write_executable_script( + &root.join("rocm"), + &tools, + &stand_in_rocm_with_an_engine_tree(&tools, &pidfile, " exit 2"), + )?; + let service_id = "svc-supervise-engine-pid-write-fails"; + register_engine_spawn_seam(service_id, true, Some(pidfile)); + + let outcome = supervise_on_stand_in_rocm(&root, &paths, service_id, 11473); + let (engine, grandchild) = take_engine_spawn_seam(service_id); + fs::remove_dir_all(&root).ok(); + + assert_engine_tree_terminated( + outcome, + service_id, + "failed to record the engine's PID", + engine, + grandchild, + ); + Ok(()) + } + + /// The write that marks the service ready fails after the engine answered + /// ready. The supervisor returns the error, and the engine tree goes with it. + #[cfg(target_os = "linux")] + #[test] + fn supervise_terminates_the_engine_when_recording_ready_fails() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("supervise-ready-write-fails"); + paths.ensure()?; + let service_id = "svc-supervise-ready-write-fails"; + let ready = serde_json::to_string(&rocm_engine_protocol::EngineResponseEnvelope::success( + rocm_engine_protocol::HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cpu".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }, + ))?; + let pidfile = root.join("engine-child.pid"); + // The readiness check spoils the manifest before it answers, so the + // engine PID write before it lands and only the ready write fails. + write_executable_script( + &root.join("rocm"), + &tools, + &stand_in_rocm_with_an_engine_tree( + &tools, + &pidfile, + &format!( + " printf 'not a service record' > '{manifest}'\n read -r _request\n \ + printf '%s\\n' '{ready}'", + manifest = paths.service_manifest_path(service_id).display(), + ), + ), + )?; + register_engine_spawn_seam(service_id, false, Some(pidfile)); + + let outcome = supervise_on_stand_in_rocm(&root, &paths, service_id, 11474); + let (engine, grandchild) = take_engine_spawn_seam(service_id); + fs::remove_dir_all(&root).ok(); + + assert_engine_tree_terminated( + outcome, + service_id, + "failed to record the service as ready", + engine, + grandchild, + ); + Ok(()) + } + + /// The engine writes its log in append mode, so a line appended to the log + /// while the engine runs is not written over by the engine's next output. + /// Here the readiness check appends the line and the engine writes after it. + #[cfg(target_os = "linux")] + #[test] + fn supervise_engine_output_does_not_overwrite_lines_appended_to_its_log() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("supervise-log-append"); + paths.ensure()?; + let service_id = "svc-supervise-log-append"; + let log_path = paths.service_log_path(service_id); + let appended = root.join("appended"); + let ready = serde_json::to_string(&rocm_engine_protocol::EngineResponseEnvelope::success( + rocm_engine_protocol::HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cpu".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }, + ))?; + write_executable_script( + &root.join("rocm"), + &tools, + &format!( + "case \"$1\" in\n\ + __engine-serve-http)\n echo engine-before\n \ + n=0\n \ + while [ ! -e '{appended}' ] && [ \"$n\" -lt 2400 ]; do '{sleep}' 0.05; n=$((n + 1)); done\n \ + echo engine-after-a-longer-line-than-the-appended-one\n ;;\n\ + __engine-stdio)\n echo appended-by-the-supervisor >> '{log}'\n : > '{appended}'\n \ + read -r _request\n printf '%s\\n' '{ready}'\n ;;\n\ + *) exit 2 ;;\nesac\n", + appended = appended.display(), + sleep = tools.sleep.display(), + log = log_path.display(), + ), + )?; + + // Registered so a run that times out kills the engine; the engine's own + // wait for the marker gives up after about two minutes regardless. + register_engine_spawn_seam(service_id, false, None); + let outcome = supervise_on_stand_in_rocm(&root, &paths, service_id, 11475); + let _ = take_engine_spawn_seam(service_id); + let log = fs::read_to_string(&log_path).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + + assert_eq!(outcome, Ok(()), "the engine exits cleanly"); + assert!( + log.contains("appended-by-the-supervisor\n"), + "the engine wrote over the appended line: {log:?}" + ); + assert!( + log.contains("engine-after-a-longer-line-than-the-appended-one\n"), + "{log:?}" + ); + Ok(()) + } + + /// Each supervised run starts the service log afresh: what an earlier run + /// left there is gone, and this run's engine output is there. + #[cfg(target_os = "linux")] + #[test] + fn supervise_empties_a_log_left_by_an_earlier_run() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("supervise-log-emptied"); + paths.ensure()?; + let service_id = "svc-supervise-log-emptied"; + let log_path = paths.service_log_path(service_id); + fs::create_dir_all(paths.services_dir())?; + fs::write(&log_path, "stale-line-from-an-earlier-run\n")?; + let ready = serde_json::to_string(&rocm_engine_protocol::EngineResponseEnvelope::success( + rocm_engine_protocol::HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cpu".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }, + ))?; + write_executable_script( + &root.join("rocm"), + &tools, + &format!( + "case \"$1\" in\n\ + __engine-serve-http)\n echo output-of-this-run\n exec '{sleep}' 1\n ;;\n\ + __engine-stdio)\n read -r _request\n printf '%s\\n' '{ready}'\n ;;\n\ + *) exit 2 ;;\nesac\n", + sleep = tools.sleep.display(), + ), + )?; + + // Registered so a run that times out kills the engine. + register_engine_spawn_seam(service_id, false, None); + let outcome = supervise_on_stand_in_rocm(&root, &paths, service_id, 11477); + let _ = take_engine_spawn_seam(service_id); + let log = fs::read_to_string(&log_path).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + + assert_eq!(outcome, Ok(()), "the engine exits cleanly"); + assert!( + !log.contains("stale-line-from-an-earlier-run"), + "the earlier run's log must be emptied: {log:?}" + ); + assert!(log.contains("output-of-this-run\n"), "{log:?}"); + Ok(()) + } + + /// If publishing the `recovering` transition fails, nothing has been + /// spawned yet: the rollback at that first write must undo the in-memory + /// transition and leave the on-disk record exactly as it was, not a + /// half-applied `recovering` row. The manifest write goes through + /// [`write_file_atomically`][rocm_core_atomic_write]-style write-then-rename, + /// so a read-only *file* no longer blocks it (rename ignores the + /// destination's own permission bits) — a read-only *directory* still does, + /// since it can't hold the new temp file the rename needs. + /// + /// [rocm_core_atomic_write]: rocm_core::atomic_write::write_file_atomically + /// + /// Unix only and skipped as root, where a read-only mode is not enforced + /// and the write would simply succeed. + #[cfg(unix)] + #[test] + fn restart_rolls_back_the_in_memory_record_when_the_first_write_fails() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if rocm_core::openmpi::running_as_root() { + return Ok(()); + } + + let (root, paths) = temp_app_paths("restart-rollback-first-write"); + paths.ensure()?; + let service_id = "svc-restart-rollback-first-write"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let services_dir = paths.services_dir(); + // The launch lock and service log files are both created lazily, on + // first acquire and on first open respectively; pre-create them so + // making the directory read-only below blocks only the manifest's + // write-then-rename, not those earlier opens. + fs::File::create(paths.managed_launch_lock_path())?; + fs::File::create(&record.log_path)?; + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; + + let result = restart_managed_service(&paths, &mut record); + + // Restore write access so the rest of the write (if any landed) can be + // read back, and so cleanup can remove the directory. + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = result.expect_err("a read-only manifest must fail the first write"); + assert!( + format!("{error:#}").contains("failed to publish the recovering service record"), + "{error:#}" + ); + assert_eq!( + record.status, "failed", + "a failed first write must restore the in-memory record rather than leave it \ + \"recovering\"" + ); + assert_eq!( + persisted?.status, "failed", + "nothing must reach disk when the first write itself fails" + ); + Ok(()) + } + + /// The first-write rollback above covers a failure before anything is + /// spawned; this one covers the second write, after the supervisor exists. + /// The spawn seam corrupts the manifest's content right after the first + /// write (`recovering`, no supervisor yet), so + /// `record_spawned_supervisor`'s read fails. That must kill the spawned + /// supervisor and restore the prior record — and the rollback write itself + /// must still land, since corrupting *content* leaves the file a writable + /// regular file. + /// + /// The spawned supervisor here is a re-exec of this very test binary, + /// which rejects its `supervise` arguments and exits on its own almost + /// immediately regardless of whether anything kills it, so asserting /// `process_is_running` on it afterward would not catch the kill being /// dropped. `TERMINATE_RECOVERY_SUPERVISOR_CALLS` asserts the call itself - /// is reached instead; `terminate_recovery_supervisor_kills_and_reaps_the_child` - /// below proves what that call does against a real long-lived process. + /// is reached instead; `terminate_recovery_supervisor_kills_and_reaps_the_child_and_its_tree` + /// above proves what that call does against a real long-lived process. #[test] fn restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read() -> Result<()> { @@ -2587,6 +4003,63 @@ mod tests { Ok(()) } + /// When the termination of the unrecorded supervisor is not confirmed, the + /// restart's error says so, naming the supervisor's PID. The rollback still + /// happens, and the supervisor is not waited for, since nothing says it will + /// exit. + #[test] + fn restart_reports_a_supervisor_it_could_not_confirm_stopped_in_its_error() -> Result<()> { + let (root, paths) = temp_app_paths("restart-unconfirmed-supervisor-termination"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-unconfirmed-supervisor-termination"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let prior = record.clone(); + + let manifest_path = paths.service_manifest_path(service_id); + let supervisor_pid = std::cell::Cell::new(None); + // As in the test above: corrupt the record between the restart's first + // write and the one recording the supervisor, so that write fails. + let corrupt = |command: &mut ProcessCommand| -> std::io::Result { + fs::write(&manifest_path, b"not json")?; + let child = command.spawn()?; + supervisor_pid.set(Some(child.id())); + Ok(child) + }; + RECOVERY_SUPERVISOR_TERMINATION_STUB + .with(|stub| stub.set(Some(rocm_core::TerminationOutcome::TimedOut))); + let outcome = restart_managed_service_with(&paths, &mut record, Some(&corrupt)); + RECOVERY_SUPERVISOR_TERMINATION_STUB.with(|stub| stub.set(None)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let pid = supervisor_pid.get().expect("the supervisor was spawned"); + let message = format!( + "{:#}", + outcome.expect_err("an unreadable second write must fail the restart") + ); + assert!( + message.contains(&format!( + "terminating the recovery supervisor (pid {pid}) ended timed_out: the recovery \ + supervisor or a process under it may still be running" + )), + "{message}" + ); + assert!( + message.contains("failed to publish the recovering service record"), + "the original failure is kept: {message}" + ); + assert_eq!( + record.status, prior.status, + "the in-memory record is restored" + ); + assert_eq!(persisted?.status, prior.status, "the rollback write landed"); + Ok(()) + } + /// As above, but the rollback write itself also fails: a manifest made /// read-only after the first write still lets `record_spawned_supervisor` /// *read* it (ownership check passes), but both its write and the diff --git a/crates/e2e-report/src/components.rs b/crates/e2e-report/src/components.rs index 4015af6173..ab008eb6a9 100644 --- a/crates/e2e-report/src/components.rs +++ b/crates/e2e-report/src/components.rs @@ -215,6 +215,8 @@ const STYLE: &str = r#" .platform-row { display: flex; align-items: center; gap: 8px; cursor: pointer; font-size: 1.05rem; } .platform-name { font-weight: 700; flex: 1; } .empty-note { color: #999; font-style: italic; padding: 8px 0; } + .lane-failure { color: #c62828; background: #ffebee; border-left: 4px solid #c62828; + padding: 8px 12px; margin: 8px 0; white-space: pre-wrap; } .scenario { border: 1px solid #e0e0e0; border-top: none; padding: 12px; background: #fff; } .scenario:last-child { border-radius: 0 0 4px 4px; } .scenario-row { display: flex; align-items: center; gap: 8px; cursor: pointer; } diff --git a/crates/e2e-report/src/consolidated/descriptor.rs b/crates/e2e-report/src/consolidated/descriptor.rs index 7291dd2cb2..5b5976d14f 100644 --- a/crates/e2e-report/src/consolidated/descriptor.rs +++ b/crates/e2e-report/src/consolidated/descriptor.rs @@ -48,6 +48,11 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { }, }; + // A lane of the shared GPU queue (see `queue_descriptor`). + if let Some(lane) = core.strip_prefix("queue-") { + return queue_descriptor(lane, channel_from_name); + } + // Legacy `-known-bugs` suffix (retained only as a stable secondary sort key). let (core, known_bugs) = match core.strip_suffix("known-bugs") { Some(rest) => (rest.trim_end_matches('-'), true), @@ -86,6 +91,68 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { } } +/// The marker an advisory queue lane carries in its platform label. +/// +/// A reader of the rendered report, and `xtask release-readiness`, which reads +/// that rendering, tell a lane the queue lets fail from one it does not by it. +pub const ADVISORY_QUEUE_MARKER: &str = "(queue, advisory)"; + +/// The GPU each shared-queue lane stands for, by the lane's name. The queue +/// discovers its lanes from its fleet, so a lane missing from this table is not +/// an error: it renders under its own name in capitals. +const QUEUE_LANE_GPUS: &[(&str, &str)] = &[ + ("mi300x", "MI300X"), + ("mi325x", "MI325X"), + ("mi350p", "MI350P"), + ("mi350x", "MI350X"), + ("mi355x", "MI355X"), + ("r9700s", "R9700"), + ("w7900-dual", "W7900"), +]; + +/// The GPU a shared-queue lane stands for (`w7900-dual` is a W7900), or the +/// lane's own name in capitals for a lane this table does not know yet. +pub fn queue_lane_gpu(lane: &str) -> String { + QUEUE_LANE_GPUS + .iter() + .find(|(name, _)| *name == lane) + .map_or_else(|| lane.to_uppercase(), |(_, gpu)| (*gpu).to_owned()) +} + +/// The artifact name `xtask e2e-queue-import` gives a queue lane's report. +/// +/// For one channel, or none for a lane that reported no channel at all. The lane +/// is named, not the `platform_slug` in its `platform.json`: queue lanes of +/// different models report the same slug (an MI325X lane probes as `mi300x`, an +/// MI355X as `mi350p`), so the slug cannot say which lane a report came from. +/// A lane the queue does not block on carries the `-advisory` suffix. +pub fn queue_artifact_name(lane: &str, advisory: bool, channel: Option<&str>) -> String { + let advisory = if advisory { "-advisory" } else { "" }; + let channel = channel.map_or_else(String::new, |c| format!("-{c}")); + format!("e2e-queue-{lane}{advisory}{channel}-report") +} + +/// Parse the core of a queue lane's artifact name (`mi325x`, `mi325x-advisory`; +/// see [`queue_artifact_name`]). Always Linux: the queue runs Linux pods. +fn queue_descriptor(lane: &str, channel_from_name: Option<&'static str>) -> Descriptor { + let (lane, advisory) = match lane.strip_suffix("-advisory") { + Some(rest) => (rest, true), + None => (lane, false), + }; + let gpu = queue_lane_gpu(lane); + let queue = if advisory { + ADVISORY_QUEUE_MARKER + } else { + "(queue)" + }; + Descriptor { + platform: format!("{gpu} {queue}"), + os: "Linux".to_owned(), + known_bugs: false, + channel_from_name, + } +} + /// The identity-bearing core of an artifact name: `e2e-` prefix and `-report` /// suffix stripped, each relative to the prior result (not `name`), so /// `e2e-report` correctly reduces to `report`, not back to itself. @@ -134,6 +201,17 @@ const KNOWN_PLATFORMS: &[KnownPlatform] = &[ os: "Linux", slugs: &["mi300x"], }, + // The same silicon, run on a self-hosted runner instead of on the shared GPU + // queue: a pull request's `e2e-gpu-report` is the queue's, so the self-hosted + // MI300X lane uploads under this name and renders beside it. No slug names it: + // the sidecar's `mi300x` is the queue's row, and a report with only a + // sidecar left cannot say which it came from. + KnownPlatform { + artifact: "e2e-gpu-selfhosted-report", + platform: "MI300X (self-hosted)", + os: "Linux", + slugs: &[], + }, // The gfx target IS the slug for this lane: `platform_hardware_slug` has no // product-name mapping for it. KnownPlatform { @@ -276,6 +354,7 @@ mod tests { for (name, platform, os) in [ ("e2e-report", "Mock", "Linux"), ("e2e-gpu-report", "MI300X", "Linux"), + ("e2e-gpu-selfhosted-report", "MI300X (self-hosted)", "Linux"), ("e2e-gpu-rad3-report", "R9700", "Linux"), ("e2e-gpu-mi350p-report", "MI350P", "Linux"), ("e2e-gpu-w7900-report", "W7900", "Linux"), @@ -328,6 +407,33 @@ mod tests { } } + #[test] + fn parse_descriptor_names_each_queue_lane_and_its_gate() { + // Every lane name is a path segment of a queue artifact, so the table is + // what turns `w7900-dual` into the GPU a reader knows. A lane the table + // lacks keeps its own name rather than being guessed at. + for (lane, gpu) in QUEUE_LANE_GPUS { + let blocking = parse_descriptor(&queue_artifact_name(lane, false, Some("release"))); + assert_eq!( + (blocking.platform.as_str(), blocking.os.as_str()), + (format!("{gpu} (queue)").as_str(), "Linux"), + "{lane}" + ); + assert_eq!(blocking.channel_from_name, Some("release")); + let advisory = parse_descriptor(&queue_artifact_name(lane, true, Some("nightly"))); + assert_eq!(advisory.platform, format!("{gpu} {ADVISORY_QUEUE_MARKER}")); + assert_eq!(advisory.channel_from_name, Some("nightly")); + } + let new_lane = parse_descriptor(&queue_artifact_name("mi400x", true, None)); + assert_eq!(new_lane.platform, "MI400X (queue, advisory)"); + assert_eq!(new_lane.channel_from_name, None); + // A queue MI300X lane is not the self-hosted MI300X lane. + assert_ne!( + parse_descriptor("e2e-queue-mi300x-report").platform, + parse_descriptor("e2e-gpu-report").platform + ); + } + #[test] fn parse_descriptor_distinguishes_sdk_version_matrix_legs() { // The sdk_version matrix (ROCMAI-430) uploads non-`current` legs under a diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs index 6c4717152f..042980bdf2 100644 --- a/crates/e2e-report/src/consolidated/html.rs +++ b/crates/e2e-report/src/consolidated/html.rs @@ -14,8 +14,9 @@ use std::path::Path; use maud::{Markup, html}; use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReport, - PlatformReports, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReport, PlatformReports, SCENARIO_ATTENTION_HEADING, + platform_cell, }; use crate::components::{feature_group, now_utc, page_shell, stats_bar}; @@ -103,6 +104,7 @@ pub fn generate_consolidated( h2 { "Platforms" } (matrix_table(reports)) (legend()) + (lane_failures_html(reports)) (expectation_grid_html(&bundle.grid)) @@ -193,7 +195,7 @@ pub(super) fn expectation_grid_html(grid: &Grid) -> Markup { } } @if !problems.is_empty() { - h3 { "Needs attention" } + h3 { (SCENARIO_ATTENTION_HEADING) } ul { @for (platform_label, id, outcome, detail) in &problems { li { @@ -246,7 +248,8 @@ fn matrix_table(reports: &[PlatformReport]) -> Markup { td.num { (fail) } td.num { (skip) } td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } } - td class=(if r.ok() { "status-pass" } else { "status-fail" }) { + td class=(if r.ok() { "status-pass" } else { "status-fail" }) + title=[r.lane_failure()] { (r.status_text()) } td { (stats_bar(&r.stats)) } @@ -288,7 +291,10 @@ fn legend() -> Markup { " — PASS when the platform is in its expected state (every " "expect-pass scenario passed and every known bug failed as " "expected); FAIL on an unexpected failure or an XPASS (a known " - "bug that unexpectedly passed — remove its @expected-failure tag)." + "bug that unexpectedly passed — remove its @expected-failure tag), " + "or when the harness failed the lane after the run (for example, no " + "scenario executed); the reason is listed under \u{201c}" + (PLATFORM_ATTENTION_HEADING) "\u{201d} and in the platform's section." } li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." } li { b { "Skip" } " — scenarios not applicable on this platform (not run)." } @@ -297,6 +303,26 @@ fn legend() -> Markup { } } +/// The platforms the harness failed after their run, with each reason: a +/// lane-check failure has no grid cell of its own (an all-skip lane has only +/// skip cells), so it is listed here. +fn lane_failures_html(reports: &[PlatformReport]) -> Markup { + let failed: Vec<(&str, &str)> = reports + .iter() + .filter_map(|r| r.lane_failure().map(|reason| (r.label.as_str(), reason))) + .collect(); + html! { + @if !failed.is_empty() { + h3 { (PLATFORM_ATTENTION_HEADING) } + ul { + @for (label, reason) in &failed { + li { b { "Lane check failed" } " in " code { (label) } ": " (reason) } + } + } + } + } +} + fn platform_section(report: &PlatformReport) -> Markup { let badge_class = if report.ok() { "badge-pass" @@ -310,6 +336,9 @@ fn platform_section(report: &PlatformReport) -> Markup { span.platform-name { (report.label) } span.elapsed { (report.stats.total) " scenarios" } } + @if let Some(reason) = report.lane_failure() { + p.lane-failure { b { "Lane check failed: " } (reason) } + } @if report.features.is_empty() { p.empty-note { "No report.json data for this platform." } } @else { @@ -328,7 +357,7 @@ mod tests { use crate::consolidated::{load_platform_reports, load_reports}; #[test] - fn legend_mock_and_other_platform_items_are_unchanged() { + fn legend_mock_and_other_platform_items_are_pinned() { // Pins the two gating items byte for byte, so building them from shared // fragments (see `MOCK_GATES_PR`) cannot change what the HTML // report's legend says. Text only: that Mock is the required check and @@ -365,12 +394,18 @@ mod tests { // now would be building the bundle's grid from `&[]` in // `load_platform_reports`. Give this one a platform.json via // write_platform and assert the grid heading actually renders. - let report = feature_json(&[(&["id:serve-a"], &["passed"])]); + // `serve-b` fails though expected to pass, so the grid's attention + // list renders too. + let report = feature_json(&[ + (&["id:serve-a"], &["passed"]), + (&["id:serve-b"], &["failed"]), + ]); let platform = r#"{ "platform_slug": "mock", "capability": {"effective_serve_engine": "none"}, "expectations": [ - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"} + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"}, + {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"} ] }"#; let (_d, path) = write_platform(&report, platform); @@ -384,6 +419,10 @@ mod tests { html.contains("Expectation grid (scenario × platform)"), "grid heading missing from output:\n{html}" ); + assert!( + html.contains(&format!("

{SCENARIO_ATTENTION_HEADING}

")), + "scenario attention heading missing from output:\n{html}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index d2d041373e..51024c330d 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -11,8 +11,9 @@ //! crate root by `lib.rs`. use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReports, - ScenarioReference, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReports, SCENARIO_ATTENTION_HEADING, ScenarioReference, + platform_cell, }; use crate::command_coverage::command_coverage_markdown; @@ -113,13 +114,26 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n", + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under {PLATFORM_ATTENTION_HEADING}).\n", ); - // Call out anything that needs a human: XPASS (fixed bug, stale tag) and + // Call out anything that needs a human: a failed post-run lane check + // (`lane_failure` in platform.json), XPASS (fixed bug, stale tag) and // untagged failures in a known-bugs run. let mut notes = Vec::new(); for r in reports { + if let Some(reason) = r.lane_failure() { + // Written by the harness's `finish_run` from `post_run_verdict`'s + // fixed strings, so not escaped; folded onto the bullet's line. + // Carrying a pre-run reason (which quotes probe stderr) into the + // report would need escaping here. + notes.push(format!( + "- **Lane check failed** in _{}_: {}", + r.label, + reason.split_whitespace().collect::>().join(" ") + )); + } for name in &r.xfail.xpass { notes.push(format!( "- **XPASS** in _{}_: `{}` is tagged `@expected-failure` but passed — remove the tag.", @@ -136,7 +150,7 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { } } if !notes.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {PLATFORM_ATTENTION_HEADING}\n\n"); for n in notes { out.push_str(&n); out.push('\n'); @@ -244,7 +258,7 @@ fn expectation_grid_markdown(grid: &Grid, scenarios: &ScenarioReference) -> Stri // Needs-attention list from reconciliation problems. let problems = grid.problems(); if !problems.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {SCENARIO_ATTENTION_HEADING}\n\n"); for (platform_label, id, outcome, detail) in problems { let bug = detail .and_then(|d| d.bug.as_deref()) @@ -360,12 +374,14 @@ mod tests { b.path().to_path_buf(), )]; let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("Needs attention")); - assert!(md.contains("XPASS")); + assert!( + md.contains("\n### Needs attention: platforms\n\n- **XPASS** in _"), + "the XPASS bullet must sit directly under the platform attention heading:\n{md}" + ); } #[test] - fn summary_explanation_paragraphs_are_unchanged() { + fn summary_explanation_paragraphs_are_pinned() { // Pins the explanatory text under the summary table byte for byte, so // building it from shared fragments (see `MOCK_GATES_PR`) cannot // change a character of what the step summary shows. This pins the text @@ -394,7 +410,12 @@ mod tests { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n"; + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under Needs attention: platforms).\n"; + // Pins the wording only. That a lane-check failure turns Status FAIL and + // is listed under "Needs attention: platforms" is proved by + // `consolidated::tests::an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason` + // and `consolidated::tests::a_recorded_lane_failure_is_named_in_the_summary_and_the_html`. assert!( md.contains(expected), "summary explanation text changed:\n{md}" @@ -410,6 +431,10 @@ mod tests { md.contains(OTHER_LANES_NON_BLOCKING), "summary must render OTHER_LANES_NON_BLOCKING ({OTHER_LANES_NON_BLOCKING:?}):\n{md}" ); + assert!( + md.contains(&format!("(listed under {PLATFORM_ATTENTION_HEADING})")), + "summary must render PLATFORM_ATTENTION_HEADING ({PLATFORM_ATTENTION_HEADING:?}):\n{md}" + ); } #[test] @@ -778,7 +803,7 @@ mod tests { let platform = r#"{ "platform_slug": "mi300x", "capability": {"effective_serve_engine": "vllm"}, - "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"11.5.1"}, + "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"2026.44.1"}, "expectations": [ {"id":"serve-x","effective_engine":"vllm","expected":"pass"} ] @@ -790,7 +815,7 @@ mod tests { "Ubuntu 24.04.3 LTS", "ROCm 7.13.0", "vLLM 0.23.0+rocm723", - "lemonade 11.5.1", + "lemonade 2026.44.1", ] { assert!(md.contains(token), "matrix cell missing {token:?}:\n{md}"); } diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 911385029d..16de2c77b6 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -24,7 +24,9 @@ mod sidecars; #[cfg(test)] mod test_support; -pub use descriptor::artifact_name_for_platform_slug; +pub use descriptor::{ + ADVISORY_QUEUE_MARKER, artifact_name_for_platform_slug, queue_artifact_name, queue_lane_gpu, +}; use descriptor::{Descriptor, full_platform_label, parse_descriptor, with_paren_suffix}; use sidecars::{ CommandRecord, ManifestExpectation, PlatformManifest, PlatformVersions, parse_commands, @@ -192,6 +194,14 @@ const MOCK_GATES_PR: &str = "gates the PR"; /// See [`MOCK_GATES_PR`]. const OTHER_LANES_NON_BLOCKING: &str = "non-blocking"; +/// Heading of the platform-level attention list (lane-check failures; in the +/// markdown summary also stale `@expected-failure` tags and regressions). +/// Distinct from [`SCENARIO_ATTENTION_HEADING`], so a report with both lists +/// never shows two identical headings. Shared by both renderers. +const PLATFORM_ATTENTION_HEADING: &str = "Needs attention: platforms"; +/// Heading of the expectation grid's per-scenario attention list. +const SCENARIO_ATTENTION_HEADING: &str = "Needs attention: scenarios"; + /// One platform column of the reconciled (scenario-id × platform) grid. struct GridColumn { /// Platform identity from the manifest (e.g. "mi300x", "strix-halo", "mock"). @@ -569,12 +579,17 @@ struct ReconciledTally { skip: u32, /// Cells that need a human: unexpected-fail, XPASS, or ran-when-N/A. problems: u32, + /// The harness failed the run after it finished (`lane_failure` in + /// `platform.json`), e.g. because no scenario executed. No cell shows that: + /// a run whose every scenario resolved to skip has only skip cells. + lane_failed: bool, } impl ReconciledTally { - /// `true` when nothing needs attention (no unexpected-fail / XPASS / ran-when-NA). + /// `true` when nothing needs attention: no unexpected-fail / XPASS / + /// ran-when-NA cell, and no failed lane check. const fn ok(&self) -> bool { - self.problems == 0 + self.problems == 0 && !self.lane_failed } const fn status_text(&self) -> &'static str { @@ -590,7 +605,10 @@ fn reconciled_tally( manifest: &PlatformManifest, actual: &std::collections::HashMap, ) -> ReconciledTally { - let mut t = ReconciledTally::default(); + let mut t = ReconciledTally { + lane_failed: manifest.lane_failure.is_some(), + ..ReconciledTally::default() + }; for exp in &manifest.expectations { match CellOutcome::reconcile(&exp.expected, exp.flaky, actual.get(&exp.id).copied()) { CellOutcome::Pass => t.pass += 1, @@ -660,6 +678,12 @@ impl PlatformReport { } } + /// Why the harness's post-run lane check failed this run, from + /// `platform.json`; `None` when it passed or the artifact predates it. + fn lane_failure(&self) -> Option<&str> { + self.manifest.as_ref()?.lane_failure.as_deref() + } + /// The channel that governs this platform's rendering. See /// [`resolve_channel`]. fn effective_channel(&self) -> Option<&str> { @@ -711,9 +735,11 @@ impl PlatformReport { /// Display counts for the summary table, reconciliation-aware so the numbers /// agree with [`Self::status_text`]. Returns `(total, pass, fail, skip, /// xfail)`. With a `platform.json`, `fail` counts only cells needing - /// attention (unexpected-fail / XPASS / ran-when-NA) — a known bug failing as - /// expected lands in `xfail`, not `fail`. Without one, falls back to raw - /// junit stats. + /// attention (unexpected-fail / XPASS / ran-when-NA / no result) — a known + /// bug failing as expected lands in `xfail`, not `fail`. A post-run lane + /// failure (`lane_failure`) has no cell, so such a row can read `FAIL` + /// with a `fail` of 0. Without a `platform.json`, falls back to raw junit + /// stats. const fn display_counts(&self) -> (u32, u32, u32, u32, u32) { if let Some(t) = &self.tally { let total = t.pass + t.xfail + t.skip + t.problems; @@ -765,6 +791,32 @@ pub struct PlatformReports { scenarios: ScenarioReference, } +/// One platform's health as the consolidated report renders it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlatformStatus { + /// Whether the row is green. + pub ok: bool, + /// The Status cell: `PASS`, `FAIL` or `EMPTY`. + pub status: &'static str, + /// The recorded lane-check failure, if any. + pub lane_failure: Option, +} + +/// Load one platform's `report.json` (and its sibling `platform.json`) the way +/// the consolidated report does, and return its status. +/// +/// Public so the harness that writes `platform.json` can test that what it +/// writes is read back as intended, keeping producer and consumer tied. +#[must_use] +pub fn platform_status(label: &str, report_json: &Path) -> PlatformStatus { + let report = PlatformReport::load(label.to_owned(), report_json); + PlatformStatus { + ok: report.ok(), + status: report.status_text(), + lane_failure: report.lane_failure().map(str::to_owned), + } +} + /// Parse every `(label, json_path)` input into a [`PlatformReport`], in input /// order. The one place `report.json`/`platform.json`/`commands.jsonl` are /// read from disk — [`Grid::build`] and [`scenario_reference`] consume the @@ -895,6 +947,237 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } + /// The artifact a lane uploads when the harness's PRE-run check refuses to + /// run (a failed capability probe, a declared GPU the probe did not find, + /// an invalid `E2E_EXPECT_GPU`): `run_suite` in + /// `tests/e2e-cucumber/tests/e2e.rs` creates `report.json` empty before it + /// calls `lane_verdict::preflight`, and exits there before writing + /// `platform.json`; on every self-hosted lane that ran its suite, the + /// report upload step (and the WSL lanes' copy-out from the guest) is + /// gated on `always()`, so that empty file is what arrives. That lane must + /// read as needing attention, never green. A POST-run lane-check failure + /// writes `platform.json` with `lane_failure` instead; see the tests below. + /// + /// The report code already behaved this way before the harness could exit + /// early, so this is not a regression test; it pins that behaviour because + /// the early exit now relies on it to keep a refused lane off green in the grid. + #[test] + fn a_lane_that_stopped_before_any_scenario_is_not_ok() { + let dir = tempfile::tempdir().expect("temp dir"); + let report = dir.path().join("report.json"); + std::fs::write(&report, "").expect("write empty report.json"); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.manifest.is_none(), "no platform.json was written"); + assert!(!r.ok(), "an empty run must not count as healthy"); + assert_eq!(r.status_text(), "EMPTY"); + } + + /// `platform.json` for a lane whose every resolved scenario was a skip, + /// with or without a recorded post-run lane failure. + fn all_skip_platform(lane_failure: Option<&str>) -> String { + let failure = lane_failure.map_or(String::new(), |reason| { + format!(r#""lane_failure": {},"#, serde_json::json!(reason)) + }); + format!( + r#"{{"platform_slug":"mi300x","capability":{{"effective_serve_engine":"vllm"}},{failure} + "expectations":[{{"id":"serve-a","feature":"Serving","expected":"skip"}}]}}"# + ) + } + + /// The regression: a GPU lane on which every scenario resolved to skip + /// was failed by the harness ("no @requires-gpu (or @requires-multi-gpu) + /// scenario executed"), yet + /// its row read PASS, since a skip cell is not a problem. + #[test] + fn an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason() { + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; + let (_dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + assert_eq!(r.lane_failure(), Some(reason)); + assert_eq!( + platform_status("e2e-gpu-report", &report), + PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + } + + /// Unchanged: an all-skip lane the harness did not fail (an undeclared or + /// `E2E_EXPECT_GPU=0` lane) stays green. + #[test] + fn an_all_skip_lane_without_a_lane_failure_still_reads_pass() { + let (dir, report) = write_platform("[]", &all_skip_platform(None)); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.ok()); + assert_eq!(r.status_text(), "PASS"); + assert_eq!(r.lane_failure(), None); + + // ...and neither rendering shows any lane-failure element. (The legend + // and the explanation paragraph name the heading in prose, so the + // checks are on the elements themselves.) + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!(!md.contains("### Needs attention: platforms"), "{md}"); + assert!(!md.contains("**Lane check failed**"), "{md}"); + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + assert!( + !html.contains("

Needs attention: platforms

"), + "{html}" + ); + assert!( + html.contains(r#"PASS"#), + "the Status cell carries no tooltip:\n{html}" + ); + assert!(!html.contains(r#"

"#), "{html}"); + } + + /// A `-n` filter that matched nothing: no expectations were resolved and + /// nothing ran, and the harness recorded "0 scenarios executed". + #[test] + fn a_zero_expectation_lane_reads_fail_only_with_a_recorded_failure() { + let failed = + r#"{"platform_slug":"mock","lane_failure":"0 scenarios executed","expectations":[]}"#; + let (_dir, report) = write_platform("[]", failed); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + + let (_dir2, report) = write_platform("[]", r#"{"platform_slug":"mock","expectations":[]}"#); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(r.ok(), "no recorded failure keeps today's reading"); + assert_eq!(r.status_text(), "PASS"); + } + + /// Both renderings name the reason, not just a FAIL. + #[test] + fn a_recorded_lane_failure_is_named_in_the_summary_and_the_html() { + // A newline, markup characters and a quote, so both the markdown's + // newline folding and the HTML escaping are exercised. + let reason = "0 scenarios executed\nsee & \"tail\""; + let html_reason = "0 scenarios executed\nsee <stderr> & "tail""; + let (dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains( + "\n### Needs attention: platforms\n\n- **Lane check failed** in _MI300X \ + Linux_: 0 scenarios executed see & \"tail\"\n" + ), + "{md}" + ); + // A CRLF, or any run of whitespace, folds to one space too. Defensive: + // no current producer of `lane_failure` emits CR or LF. + let (_crlf_dir, crlf_report) = + write_platform("[]", &all_skip_platform(Some("first\r\n second"))); + let crlf_md = markdown::consolidated_summary_markdown(&load_platform_reports(&[( + "e2e-gpu-report".to_owned(), + crlf_report, + )])); + assert!(crlf_md.contains("Linux_: first second\n"), "{crlf_md}"); + + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + // The Status cell's tooltip. + assert!( + html.contains(&format!( + r#"FAIL"# + )), + "status tooltip:\n{html}" + ); + // The platform section's paragraph. + assert!( + html.contains(&format!( + r#"

Lane check failed: {html_reason}

"# + )), + "lane failure paragraph:\n{html}" + ); + // The platform-level attention list. + assert!( + html.contains(&format!( + r"

Needs attention: platforms

  • Lane check failed in MI300X Linux: {html_reason}
" + )), + "attention list:\n{html}" + ); + // ...styled, so it reads as a failure. + assert!( + html.contains(".lane-failure {"), + "CSS rule for .lane-failure" + ); + // The legend says where the reason is. + assert!( + html.contains( + "or when the harness failed the lane after the run (for example, no scenario \ + executed); the reason is listed under \u{201c}Needs attention: platforms\u{201d} \ + and in the platform's section." + ), + "legend:\n{html}" + ); + } + + /// A lane on which scenarios ran and passed, but which the harness still + /// failed (on a GPU lane, no GPU scenario among them): its row + /// reads FAIL with a Fail count of 0, since no cell failed. Rows of exactly + /// this shape are what `total_e2e_failures` in xtask counts as failures. + #[test] + fn a_lane_that_ran_scenarios_and_was_failed_reads_fail_with_zero_failed_cells() { + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (3 resolved, none ran)"; + let platform = format!( + r#"{{"platform_slug":"mi300x","lane_failure":{},"expectations":[ + {{"id":"examine-a","feature":"F","expected":"pass"}}]}}"#, + serde_json::json!(reason) + ); + let (_dir, report) = write_platform( + &feature_json(&[(&["id:examine-a"], &["passed"])]), + &platform, + ); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert_eq!(r.display_counts(), (1, 1, 0, 0, 0)); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains("| MI300X | Linux | 1 | 1 | 0 | 0 | n/a | FAIL |"), + "{md}" + ); + assert!(md.contains("| **Total** | | 1 | 1 | 0 | 0 | 0 | |"), "{md}"); + } + + /// Both attention lists can appear in one summary, under different + /// headings, each once. + #[test] + fn the_two_attention_lists_have_distinct_headings() { + let platform = r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#; + let (_dir, report) = + write_platform(&feature_json(&[(&["id:examine-a"], &["failed"])]), platform); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert_eq!( + md.matches("### Needs attention: platforms\n").count(), + 1, + "{md}" + ); + assert_eq!( + md.matches("### Needs attention: scenarios\n").count(), + 1, + "{md}" + ); + assert_eq!(md.matches("### Needs attention\n").count(), 0, "{md}"); + } + #[test] fn rendering_uses_the_bundle_not_the_artifacts_on_disk() { // The render functions must work from what `load_platform_reports` @@ -1763,8 +2046,8 @@ mod tests { #[test] fn renderers_take_the_pr_gating_phrases_from_the_shared_constants() { // What this proves, and what it does not. The rendered text is pinned - // verbatim by `summary_explanation_paragraphs_are_unchanged` (markdown) - // and `legend_mock_and_other_platform_items_are_unchanged` (HTML), and + // verbatim by `summary_explanation_paragraphs_are_pinned` (markdown) + // and `legend_mock_and_other_platform_items_are_pinned` (HTML), and // each of those also asserts that the rendered output contains // `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`; together they catch a // renderer that hardcodes the words while a constant's value moves on. diff --git a/crates/e2e-report/src/consolidated/sidecars.rs b/crates/e2e-report/src/consolidated/sidecars.rs index b1ff323864..28de5088e9 100644 --- a/crates/e2e-report/src/consolidated/sidecars.rs +++ b/crates/e2e-report/src/consolidated/sidecars.rs @@ -54,6 +54,10 @@ pub(super) struct PlatformManifest { /// Component versions (OS/ROCm/vLLM/lemonade); absent in older artifacts. #[serde(default)] pub(super) versions: PlatformVersions, + /// Why the harness's post-run lane check failed the run (for example "0 + /// scenarios executed"); absent when it passed, and in older artifacts. + #[serde(default)] + pub(super) lane_failure: Option, #[serde(default)] pub(super) expectations: Vec, } diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 73a702e740..448e9d3941 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -16,7 +16,10 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; -pub use consolidated::{PlatformReports, artifact_name_for_platform_slug, load_platform_reports}; +pub use consolidated::{ + ADVISORY_QUEUE_MARKER, PlatformReports, PlatformStatus, artifact_name_for_platform_slug, + load_platform_reports, platform_status, queue_artifact_name, queue_lane_gpu, +}; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; @@ -33,3 +36,18 @@ pub use single_report::generate; /// typo previously would not fail to compile or fail a test, silently turning /// `@requires-oom-fault-injection` into a skip (green suite, zero coverage). pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION"; + +/// Environment variable in which a CI lane declares whether it has a GPU. +/// +/// `1` means the lane runs on GPU hardware, so the `e2e-cucumber` harness fails +/// before any scenario runs when its capability probe finds no usable AMD GPU, +/// and after the run when no `@requires-gpu` (or `@requires-multi-gpu`) +/// scenario executed. `0` means the lane deliberately has no usable GPU (the +/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or +/// empty leaves the lane undeclared, as on the GitHub-hosted lanes and local +/// runs; any other value is an error. +/// +/// Shared by the harness, which reads it, and xtask's workflow contract test, +/// which checks that every self-hosted E2E lane sets it, so the two cannot name +/// different variables. +pub const EXPECT_GPU_ENV: &str = "E2E_EXPECT_GPU"; diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index aaaa52c643..f6b7d64c41 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -14,9 +14,11 @@ workspace = true # Lets the E2E suite re-root the hardware probes under a simulated host; see # `src/hardware_root.rs`. Never enabled in a release build. e2e-test-hooks = [] -# Compiles the `foreign_temp_probe` test probe so a dependent crate's tests can -# use it. Enabled only from a `[dev-dependencies]` entry; never in a shipped -# build. +# Compiles test helpers so a dependent crate's tests can use them: the +# `foreign_temp_probe` probe, `atomic_write::stage_owner_only_file_unpublished` +# (the leftover a killed owner-only store leaves) and, on Windows, +# `owner_only_file::owner_only_dacl_mismatch` (reads a file's DACL back). +# Enabled only from a `[dev-dependencies]` entry; never in a shipped build. test-support = [] [build-dependencies] @@ -70,4 +72,4 @@ tempfile = "3" chrono.workspace = true [target.'cfg(target_os = "windows")'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_Registry", "Win32_System_SystemInformation", "Win32_System_Threading", "Win32_UI_WindowsAndMessaging"] } +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Registry", "Win32_System_SystemInformation", "Win32_System_Threading", "Win32_UI_WindowsAndMessaging"] } diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index e63f3e70b8..ef7616de33 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -28,10 +28,21 @@ //! //! A publish by `rename` leaves the destination with the staged file's //! attributes and the ACL its directory hands down, where `ReplaceFileW` carries -//! over those of the file it replaces. No caller of this module sets a hidden or -//! read-only flag or a custom ACL on a file it writes, so nothing is lost today. -//! A file that must keep such settings across a rewrite should not rely on this -//! writer to carry them over. +//! over those of the file it replaces. No caller of [`write_file_atomically`] +//! sets a hidden or read-only flag or a custom ACL on a file it writes, so +//! nothing is lost today. A file that must keep such settings across a rewrite +//! should not rely on this writer to carry them over. +//! +//! [`write_owner_only_file_atomically`] is the variant for secrets. It stages +//! the bytes in a file only the current user can open (see +//! [`crate::owner_only_file`]) instead of copying the old file's permissions, +//! and publishes by `rename` alone on every platform, never `ReplaceFileW`, +//! because `ReplaceFileW` would carry the old file's ACL — possibly a loose +//! one — over to the new contents. A destination another process holds open +//! without delete sharing therefore makes it fail rather than fall back. It +//! does not resolve a symbolic link at the destination either: the rename +//! replaces the link with the new file instead of writing the secret through +//! it to wherever it points. //! //! The fallback is the one path that can leave the destination absent: //! Microsoft documents that `ReplaceFileW`, called without a backup name, can @@ -50,7 +61,8 @@ //! the old file or the new one rather than an empty one. Windows has no //! directory flush here; there the rename's durability is the filesystem's. //! -//! Replacing a file this way must not quietly change what the user set up: +//! Replacing a file with [`write_file_atomically`] must not quietly change what +//! the user set up: //! //! - on Unix the new file keeps the old one's permission bits, applied to the //! temp file before any byte is written into it (on Windows nothing is @@ -201,6 +213,10 @@ where rename(replacement, path).map_err(|retry| destination_removed_error(error, retry)) } +/// What separates a staged file's name from the name of the file it will +/// replace: `{file name}.tmp-{suffix}`. +const STAGED_NAME_MARKER: &str = ".tmp-"; + /// A temp path next to `path`, preserving the full file name so a /// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes /// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). @@ -210,15 +226,90 @@ fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { .file_name() .context("file path has no file name")? .to_os_string(); - file_name.push(".tmp-"); + file_name.push(STAGED_NAME_MARKER); file_name.push(suffix); Ok(parent.join(file_name)) } +/// The name of the file a staged file named `file_name` was going to replace, +/// or `None` when `file_name` is not a staged file's name. +/// +/// A writer killed between staging and publishing leaves its staged file +/// behind, and for a secret written with [`write_owner_only_file_atomically`] +/// that file holds the secret. This lets a cleanup that removes the +/// destination recognise those leftovers as belonging to it. +/// +/// Only the suffix this module's own writers produce is recognised — +/// `{pid}-{millis}-{attempt}`, three runs of digits — so a file whose name +/// merely contains `.tmp-` is not taken for a staged one. That matters where +/// names are chosen by someone else: the service id `a.endpoint-key.tmp-x` +/// gives a key file named `a.endpoint-key.tmp-x.endpoint-key`, which must not +/// read as a staged copy of service `a`'s key. Names are compared as text, so +/// a staged file whose name is not Unicode is not recognised. +pub fn staged_destination_name(file_name: &str) -> Option<&str> { + let (destination, suffix) = file_name.rsplit_once(STAGED_NAME_MARKER)?; + let mut parts = suffix.split('-'); + let well_formed = (0..3).all(|_| { + parts + .next() + .is_some_and(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())) + }) && parts.next().is_none(); + (!destination.is_empty() && well_formed).then_some(destination) +} + +/// Every staged file next to `path` that was going to replace it — what a +/// writer killed before publishing leaves behind. Empty when there are none or +/// the directory cannot be read. +/// +/// A writer that is still running has its staged file listed too, so remove +/// these only when the destination itself is being removed. The result is a +/// snapshot taken when the directory is read; it does not exclude a concurrent +/// writer. +pub fn staged_siblings(path: &Path) -> Vec { + let (Some(parent), Some(name)) = (path.parent(), path.file_name()) else { + return Vec::new(); + }; + // A bare relative name has the empty path as its parent, which cannot be + // read; its folder is the current one. Each result is `parent` joined with + // the staged file's name, so it is relative exactly when `path` is. + let folder = if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + }; + let Ok(entries) = fs::read_dir(folder) else { + return Vec::new(); + }; + let mut staged: Vec = entries + .flatten() + .map(|entry| entry.file_name()) + .filter(|candidate| { + candidate + .to_str() + .and_then(staged_destination_name) + .is_some_and(|destination| OsStr::new(destination) == name) + }) + .map(|candidate| parent.join(candidate)) + .collect(); + staged.sort(); + staged +} + fn process_temp_id() -> String { format!("{}-{}", std::process::id(), unix_time_millis()) } +/// How a staged file gets its permissions before any byte is written into it. +#[derive(Clone, Copy, Debug)] +enum StagingMode { + /// Copy the permission bits of the file it will replace, if any (Unix + /// only; see `keep_existing_permissions`). + KeepExisting, + /// Create it readable and writable by the current user only, whatever the + /// file it replaces allowed. + OwnerOnly, +} + /// Replace `path` with `bytes` atomically, creating missing parent directories. pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { let temp_id = process_temp_id(); @@ -280,11 +371,56 @@ where F: FnOnce(&Path, &Path) -> io::Result<()>, { let path = resolve_symlink_target(path)?; - let tmp = stage_file_for_atomic_publish_with(&path, bytes, suffix_for_attempt)?; + let tmp = stage_file_for_atomic_publish_with( + &path, + bytes, + suffix_for_attempt, + StagingMode::KeepExisting, + )?; before_publish(); publish_staged_file_with(&tmp, &path, publish) } +/// Replace `path` with `bytes` atomically, as a file only the current user can open. +/// +/// Mode `0600` on Unix, a protected DACL granting only the current user on +/// Windows. Creates missing parent directories. +/// +/// Unlike [`write_file_atomically`], nothing of the file being replaced is +/// kept: a key file someone loosened to `0644`, or one that inherited a +/// readable ACL, comes back owner-only. The publish is a plain rename on every +/// platform, so it fails, leaving the old file in place, when another process +/// holds the destination open without delete sharing. A symbolic link at +/// `path` is replaced, not followed. See the module doc. +pub fn write_owner_only_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let tmp = stage_owner_only_file(path, bytes)?; + publish_staged_file_with(&tmp, path, rename_file) +} + +/// The staging half of [`write_owner_only_file_atomically`]. +fn stage_owner_only_file(path: &Path, bytes: &[u8]) -> Result { + let temp_id = process_temp_id(); + stage_file_for_atomic_publish_with( + path, + bytes, + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + StagingMode::OwnerOnly, + ) +} + +/// Test support: what a [`write_owner_only_file_atomically`] killed between +/// staging and publishing leaves behind — `bytes` in an owner-only staged file +/// next to `path`, never published. Returns the staged file's path. +/// +/// Compiled only for this crate's tests and under the `test-support` feature, +/// so the tests of the cleanups that must remove such leftovers start from the +/// file a real store leaves rather than a hand-made look-alike. +#[cfg(any(test, feature = "test-support"))] +#[doc(hidden)] +pub fn stage_owner_only_file_unpublished(path: &Path, bytes: &[u8]) -> Result { + stage_owner_only_file(path, bytes) +} + /// The file a write to `path` should replace: `path` itself, or — when it is /// a symbolic link — the file the link chain ends at, so the link survives. /// A dangling link resolves to where its target would be, which is created. @@ -326,15 +462,19 @@ fn resolve_symlink_target(path: &Path) -> Result { /// publishing it over that `path` replaces the link with a regular file. pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { let temp_id = process_temp_id(); - stage_file_for_atomic_publish_with(path, bytes, |attempt| { - OsString::from(format!("{temp_id}-{attempt}")) - }) + stage_file_for_atomic_publish_with( + path, + bytes, + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + StagingMode::KeepExisting, + ) } fn stage_file_for_atomic_publish_with( path: &Path, bytes: &[u8], suffix_for_attempt: S, + mode: StagingMode, ) -> Result where S: FnMut(u32) -> OsString, @@ -342,9 +482,13 @@ where // Flushed before it can be published: a rename that reaches the disk // before the data would leave an empty file under the real name after a // crash. A full disk can surface in the flush rather than in `write_all`. - stage_file_for_atomic_publish_with_writer(path, bytes, suffix_for_attempt, |file, bytes| { - write_then_flush(file, bytes, fs::File::sync_all) - }) + stage_file_for_atomic_publish_with_writer( + path, + bytes, + suffix_for_attempt, + mode, + |file, bytes| write_then_flush(file, bytes, fs::File::sync_all), + ) } /// [`stage_file_for_atomic_publish_with`] with the write into the reserved @@ -354,6 +498,7 @@ fn stage_file_for_atomic_publish_with_writer( path: &Path, bytes: &[u8], mut suffix_for_attempt: S, + mode: StagingMode, write: W, ) -> Result where @@ -367,11 +512,14 @@ where for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { let suffix = suffix_for_attempt(attempt); let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { + let opened = match mode { + StagingMode::KeepExisting => fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp), + StagingMode::OwnerOnly => crate::owner_only_file::open_new_owner_only(&tmp), + }; + match opened { Ok(file) => { reserved = Some((tmp, file)); break; @@ -391,8 +539,13 @@ where }; // Before any byte is written, so a private file's new contents are never - // readable under the wider default mode, even for a moment. - if let Err(error) = keep_existing_permissions(path, &tmp) { + // readable under the wider default mode, even for a moment. An owner-only + // file was created narrow and must not be widened to the old file's mode. + let kept = match mode { + StagingMode::KeepExisting => keep_existing_permissions(path, &tmp), + StagingMode::OwnerOnly => Ok(()), + }; + if let Err(error) = kept { drop(file); let _ = fs::remove_file(&tmp); return Err(error); @@ -524,6 +677,24 @@ fn sync_parent_dir(path: &Path) { #[allow(clippy::missing_const_for_fn)] fn sync_parent_dir(_path: &Path) {} +#[cfg(test)] +thread_local! { + /// How many times this thread has entered a publish step that can reach + /// `ReplaceFileW`: `publish_rename_first`, which falls back to it, and + /// `replace_file_windows`, which calls it — and, where neither exists, the + /// non-Windows [`publish_temp_file`] that stands in for them. Counted where + /// the fallback lives rather than at one caller, so any route to it is + /// seen, and on every platform, so a test on any host can check that + /// [`write_owner_only_file_atomically`] never takes it. + static REPLACE_CAPABLE_PUBLISHES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Test-only: count one entry into a publish step that can reach `ReplaceFileW`. +#[cfg(test)] +fn note_replace_capable_publish() { + REPLACE_CAPABLE_PUBLISHES.with(|count| count.set(count.get() + 1)); +} + /// Test seam: move the staged file `tmp` over `path` in one step, replacing /// any existing file without first removing it. /// @@ -533,6 +704,8 @@ fn sync_parent_dir(_path: &Path) {} #[cfg(not(windows))] #[doc(hidden)] pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + #[cfg(test)] + note_replace_capable_publish(); fs::rename(tmp, path) } @@ -559,6 +732,8 @@ where R: FnOnce(&Path, &Path) -> io::Result<()>, P: FnOnce(&Path, &Path) -> io::Result<()>, { + #[cfg(test)] + note_replace_capable_publish(); match rename(tmp, path) { Ok(()) => Ok(()), Err(rename_error) => { @@ -577,6 +752,9 @@ fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + #[cfg(test)] + note_replace_capable_publish(); + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); let replacement_wide: Vec = replacement .as_os_str() @@ -611,11 +789,11 @@ fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { recover_partial_replace(io::Error::last_os_error(), replacement, path, rename_file) } -/// `fs::rename` with both paths borrowed, the shape `recover_partial_replace` -/// and `publish_rename_first` take. `fs::rename` itself is generic over -/// `AsRef`, so passed directly it is pinned to one lifetime per argument -/// and does not satisfy their `FnOnce(&Path, &Path)` bounds. -#[cfg(windows)] +/// `fs::rename` with both paths borrowed, the shape `recover_partial_replace`, +/// `publish_rename_first` and `publish_staged_file_with` take. `fs::rename` +/// itself is generic over `AsRef`, so passed directly it is pinned to one +/// lifetime per argument and does not satisfy their `FnOnce(&Path, &Path)` +/// bounds. fn rename_file(from: &Path, to: &Path) -> io::Result<()> { fs::rename(from, to) } @@ -648,6 +826,7 @@ mod tests { &destination, b"payload", |attempt| OsString::from(format!("full-disk-{attempt}")), + StagingMode::KeepExisting, |_, _| Err(io::Error::from(io::ErrorKind::StorageFull)), ) .expect_err("a write that hits a full disk must fail"); @@ -683,10 +862,15 @@ mod tests { fs::write(&taken, b"another writer's staged file").unwrap(); let mut tries = 0; - let error = stage_file_for_atomic_publish_with(&destination, b"payload", |_| { - tries += 1; - OsString::from("taken") - }) + let error = stage_file_for_atomic_publish_with( + &destination, + b"payload", + |_| { + tries += 1; + OsString::from("taken") + }, + StagingMode::KeepExisting, + ) .expect_err("a writer that finds every name taken must fail"); let foreign = fs::read(&taken); @@ -969,14 +1153,18 @@ mod tests { // Both threads ask for the same first name, as two writers // sharing a pid and a millisecond would, so the collision // happens every run rather than when the clock lines up. - let staged = - stage_file_for_atomic_publish_with(&destination, bytes, |attempt| { + let staged = stage_file_for_atomic_publish_with( + &destination, + bytes, + |attempt| { if attempt == 0 { OsString::from("same-millisecond") } else { OsString::from(format!("same-millisecond-{writer}-{attempt}")) } - })?; + }, + StagingMode::KeepExisting, + )?; before_publish.wait(); publish_staged_file(&staged, &destination) }) @@ -1340,7 +1528,6 @@ mod tests { } /// The names in `dir`, sorted. - #[cfg(unix)] fn names_in(dir: &Path) -> Vec { let mut names: Vec = fs::read_dir(dir) .unwrap() @@ -1394,6 +1581,7 @@ mod tests { &destination, b"secret", |attempt| OsString::from(format!("mode-{attempt}")), + StagingMode::KeepExisting, |file, bytes| { mode_while_writing = Some(file.metadata()?.permissions().mode() & 0o777); file.write_all(bytes) @@ -1562,4 +1750,305 @@ mod tests { ); assert_eq!(names, ["a.json", "b.json"]); } + + /// What a writer killed before publishing leaves — a file the real staging + /// path names — is found as belonging to its destination, and nothing + /// else is: not the destination, not another file's staged copy whose + /// name merely starts the same way. + #[test] + fn staged_siblings_finds_the_files_staging_left_for_that_destination_only() { + let root = scratch_dir("staged-siblings"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("svc.endpoint-key"); + fs::write(&destination, b"published").unwrap(); + let staged = stage_owner_only_file_unpublished(&destination, b"secret").unwrap(); + for unrelated in [ + "svc.endpoint-key2.tmp-1-2-0", + "other-svc.endpoint-key.tmp-1-2-0", + "svc.endpoint-key.tmp-", + "svc.log", + ] { + fs::write(root.join(unrelated), b"x").unwrap(); + } + + let found = staged_siblings(&destination); + let _ = fs::remove_dir_all(&root); + assert_eq!(found, [staged]); + } + + /// The name the real owner-only staging writes is read back as its + /// destination's; a name that merely contains the marker is not. + #[test] + fn staged_destination_name_reads_back_only_what_staging_writes() { + let root = scratch_dir("staged-name"); + let destination = root.join("svc-1.endpoint-key"); + let staged = stage_owner_only_file_unpublished(&destination, b"secret").unwrap(); + let name = staged.file_name().unwrap().to_str().unwrap().to_owned(); + let _ = fs::remove_dir_all(&root); + + assert_eq!(staged_destination_name(&name), Some("svc-1.endpoint-key")); + assert_eq!(staged_destination_name("svc-1.endpoint-key"), None); + assert_eq!(staged_destination_name(".tmp-1-2-0"), None); + assert_eq!(staged_destination_name("svc-1.endpoint-key.tmp-"), None); + assert_eq!(staged_destination_name("svc-1.endpoint-key.tmp-1-2"), None); + assert_eq!( + staged_destination_name("svc-1.endpoint-key.tmp-1-2-0-3"), + None + ); + assert_eq!( + staged_destination_name("svc-1.endpoint-key.tmp-1-x-0"), + None + ); + // Three runs, but one of them empty: not a suffix staging writes. + for empty_run in [ + "x.tmp-1--3", + "x.tmp--2-3", + "x.tmp-1-2-", + "x.tmp---", + "x.tmp--", + ] { + assert_eq!(staged_destination_name(empty_run), None, "{empty_run}"); + } + // The key file of a service whose id itself contains the marker. + assert_eq!( + staged_destination_name("a.endpoint-key.tmp-x.endpoint-key"), + None + ); + assert_eq!( + staged_destination_name("a.endpoint-key.tmp-1-2-3.endpoint-key"), + None + ); + } + + /// Cleaning up service `a`'s key must not take the live key of a service + /// whose id was chosen to look like a staged copy of it. + #[test] + fn staged_siblings_ignores_a_key_named_like_a_staged_copy() { + let root = scratch_dir("staged-adversarial"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("a.endpoint-key"); + fs::write(&destination, b"a's key").unwrap(); + for id in ["a.endpoint-key.tmp-x", "a.endpoint-key.tmp-1-2-3"] { + fs::write(root.join(format!("{id}.endpoint-key")), b"another key").unwrap(); + } + + let found = staged_siblings(&destination); + let _ = fs::remove_dir_all(&root); + assert!(found.is_empty(), "{found:?}"); + } + + /// A destination given as a bare relative name has its staged copies found + /// in the current directory, and each comes back as a bare name too. + /// + /// The current directory is process-wide, so the bare-name case runs in a + /// re-executed copy of this test started in a scratch directory, rather + /// than here, where changing it would reach sibling tests and leaving it + /// would write into the source tree. + #[test] + fn staged_siblings_finds_the_copies_of_a_bare_relative_name() { + const NAME: &str = "svc.endpoint-key"; + if std::env::var_os("ROCM_TEST_STAGED_RELATIVE_CHILD").is_some() { + let destination = PathBuf::from(NAME); + fs::write(&destination, b"published").unwrap(); + let staged = stage_owner_only_file_unpublished(&destination, b"secret").unwrap(); + assert_eq!(staged.parent(), Some(Path::new("")), "{staged:?}"); + assert_eq!(staged_siblings(&destination), [staged]); + return; + } + + let root = scratch_dir("staged-relative"); + fs::create_dir_all(&root).unwrap(); + let status = + std::process::Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "atomic_write::tests::staged_siblings_finds_the_copies_of_a_bare_relative_name", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_STAGED_RELATIVE_CHILD", "1") + .current_dir(&root) + .status() + .expect("re-exec the test in a child process"); + let ran = root.join(NAME).exists(); + let _ = fs::remove_dir_all(&root); + assert!(ran, "the child did not run the test arm"); + assert!(status.success(), "{status:?}"); + } + + /// The owner-only writer never enters a publish step that can reach + /// `ReplaceFileW`, which would carry the replaced file's DACL over to the + /// secret. Checked on every host by counting entries into those steps, + /// which [`write_file_atomically`] shows are counted. + #[test] + fn write_owner_only_file_atomically_never_takes_the_publish_with_the_replace_fallback() { + let count = || REPLACE_CAPABLE_PUBLISHES.with(std::cell::Cell::get); + let root = scratch_dir("owner-only-no-replace"); + let destination = root.join("svc.endpoint-key"); + + let before = count(); + write_owner_only_file_atomically(&destination, b"new").unwrap(); + write_owner_only_file_atomically(&destination, b"over an existing file").unwrap(); + let owner_only = count() - before; + + let before = count(); + write_file_atomically(&root.join("other.json"), b"{}").unwrap(); + let general = count() - before; + let _ = fs::remove_dir_all(&root); + + // At least one: on Windows a rename that fails falls through to the + // replace, which counts again. + assert!(general >= 1, "the count does not see the general publish"); + assert_eq!( + owner_only, 0, + "the owner-only writer published through the ReplaceFileW-capable path" + ); + } + + /// A secret file someone loosened comes back owner-only: unlike + /// [`write_file_atomically`], the owner-only writer does not keep the old + /// mode, and does not leave a temp file behind. + #[cfg(unix)] + #[test] + fn write_owner_only_file_atomically_tightens_a_loosened_file() { + use std::os::unix::fs::PermissionsExt as _; + + let root = scratch_dir("owner-only-loosened"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("svc.endpoint-key"); + fs::write(&destination, b"old").unwrap(); + fs::set_permissions(&destination, fs::Permissions::from_mode(0o644)).unwrap(); + + write_owner_only_file_atomically(&destination, b"new").unwrap(); + + let mode = fs::metadata(&destination).unwrap().permissions().mode() & 0o777; + let contents = fs::read(&destination).unwrap(); + let names = names_in(&root); + let _ = fs::remove_dir_all(&root); + assert_eq!(mode, 0o600, "mode was {mode:o}"); + assert_eq!(contents, b"new"); + assert_eq!(names, ["svc.endpoint-key"]); + } + + /// A new file, in a directory that does not exist yet, is created 0600. + #[cfg(unix)] + #[test] + fn write_owner_only_file_atomically_creates_a_new_file_0600() { + use std::os::unix::fs::PermissionsExt as _; + + let root = scratch_dir("owner-only-new"); + let destination = root.join("services").join("svc.endpoint-key"); + + write_owner_only_file_atomically(&destination, b"key").unwrap(); + + let mode = fs::metadata(&destination).unwrap().permissions().mode() & 0o777; + let contents = fs::read(&destination).unwrap(); + let _ = fs::remove_dir_all(&root); + assert_eq!(mode, 0o600, "mode was {mode:o}"); + assert_eq!(contents, b"key"); + } + + /// A symbolic link at the destination is replaced by the new file, not + /// followed: the secret must not be written to wherever the link points, + /// and the link's target keeps its contents and mode. + #[cfg(unix)] + #[test] + fn write_owner_only_file_atomically_replaces_a_symlink_instead_of_following_it() { + use std::os::unix::fs::{PermissionsExt as _, symlink}; + + let root = scratch_dir("owner-only-symlink"); + fs::create_dir_all(&root).unwrap(); + let target = root.join("elsewhere"); + fs::write(&target, b"untouched").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o644)).unwrap(); + let destination = root.join("svc.endpoint-key"); + symlink(&target, &destination).unwrap(); + + write_owner_only_file_atomically(&destination, b"secret").unwrap(); + + let is_link = fs::symlink_metadata(&destination) + .unwrap() + .file_type() + .is_symlink(); + let contents = fs::read(&destination).unwrap(); + let mode = fs::metadata(&destination).unwrap().permissions().mode() & 0o777; + let target_contents = fs::read(&target).unwrap(); + let target_mode = fs::metadata(&target).unwrap().permissions().mode() & 0o777; + let _ = fs::remove_dir_all(&root); + assert!(!is_link, "the link was kept and followed"); + assert_eq!(contents, b"secret"); + assert_eq!(mode, 0o600, "mode was {mode:o}"); + assert_eq!(target_contents, b"untouched"); + assert_eq!(target_mode, 0o644); + } + + /// A publish that cannot rename over the destination fails, leaves the + /// destination alone and removes the staged file, so no stray copy of the + /// secret stays on disk. + #[test] + fn write_owner_only_file_atomically_cleans_up_when_the_rename_fails() { + let root = scratch_dir("owner-only-rename-fails"); + let destination = root.join("svc.endpoint-key"); + // A non-empty directory cannot be replaced by a rename on any platform. + fs::create_dir_all(destination.join("occupied")).unwrap(); + + let error = write_owner_only_file_atomically(&destination, b"secret") + .expect_err("renaming a file over a directory must fail"); + + let names = names_in(&root); + let still_a_dir = destination.join("occupied").is_dir(); + let _ = fs::remove_dir_all(&root); + assert!( + format!("{error:#}").contains("failed to publish"), + "{error:#}" + ); + assert_eq!(names, ["svc.endpoint-key"]); + assert!(still_a_dir); + } + + /// On Windows a destination that inherited or was given a loose DACL comes + /// back with a protected DACL granting only the current user, because the + /// publish is a rename of a file created owner-only, never a + /// `ReplaceFileW` that would carry the old DACL over. + #[cfg(windows)] + #[test] + fn write_owner_only_file_atomically_replaces_a_loose_dacl_with_an_owner_only_one() { + use crate::owner_only_file::owner_only_sddl; + use crate::owner_only_file::win32::test_support::{ + canonical_sddl, dacl_sddl, flags_and_entries, + }; + use crate::owner_only_file::win32::{create_new_with_sddl, current_user_sid}; + + let root = scratch_dir("owner-only-dacl"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("svc.endpoint-key"); + let sid = current_user_sid().unwrap(); + // Everyone (`WD`) may read and write it, and it inherits too. + let loose = format!("D:(A;;FA;;;WD)(A;;FA;;;{sid})"); + let mut file = create_new_with_sddl(&destination, &loose).unwrap(); + file.write_all(b"old").unwrap(); + drop(file); + let before = dacl_sddl(&destination).unwrap(); + + write_owner_only_file_atomically(&destination, b"new").unwrap(); + + let after = dacl_sddl(&destination).unwrap(); + let expected = canonical_sddl(&owner_only_sddl(&sid)).unwrap(); + let contents = fs::read(&destination).unwrap(); + let names = names_in(&root); + let _ = fs::remove_dir_all(&root); + + assert!( + before.contains(";;;WD)"), + "the setup DACL is not loose: {before}" + ); + let (flags, entries) = flags_and_entries(&after); + let (_, expected_entries) = flags_and_entries(&expected); + assert!(flags.contains('P'), "the DACL is not protected: {after}"); + assert_eq!( + entries, expected_entries, + "the DACL grants more than the user" + ); + assert_eq!(contents, b"new"); + assert_eq!(names, ["svc.endpoint-key"]); + } } diff --git a/crates/rocm-core/src/diagnose/wsl.rs b/crates/rocm-core/src/diagnose/wsl.rs index d34fa9644f..2edd502a6b 100644 --- a/crates/rocm-core/src/diagnose/wsl.rs +++ b/crates/rocm-core/src/diagnose/wsl.rs @@ -266,7 +266,7 @@ pub(super) fn check_wsl_5_distro_too_old(e: &Examination, _symptom: &str) -> Dia verify: "f=/etc/os-release; [ -e \"$f\" ] || f=/usr/lib/os-release; grep VERSION_ID \"$f\" 2>/dev/null | head -1" .to_owned(), notes: vec![ - "This is a hard floor, not a recommendation: Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 that every published Lemonade embeddable is linked against, so the engine cannot start there at all.".to_owned(), + "This is a hard floor, not a recommendation: Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 that the pinned Lemonade embeddable is linked against (newer Lemonade releases need more), so the engine cannot start there at all.".to_owned(), ], ..Fix::default() }; @@ -660,7 +660,7 @@ mod tests { #[test] fn a_rocdxg_beside_the_runtime_is_not_reported_as_unlinked() { // The ROCM-30386 layout: a TheRock tarball keeps librocdxg.so beside - // libhsa-runtime64, which opens it through its own `$ORIGIN` runpath. + // libhsa-runtime64, which opens it through its own `$ORIGIN` search path. let mut e = wsl_base(); let w = e.wsl.as_mut().expect("wsl facts"); w.ldconfig_librocdxg = Some(false); diff --git a/crates/rocm-core/src/endpoint_readiness.rs b/crates/rocm-core/src/endpoint_readiness.rs index fe4eec3a5e..52895a8bec 100644 --- a/crates/rocm-core/src/endpoint_readiness.rs +++ b/crates/rocm-core/src/endpoint_readiness.rs @@ -11,17 +11,24 @@ //! (`engine_state_inference_verified`), and the model-reference matching that //! decides whether a listed model id is the one a service was started with. //! +//! The engine state file that latch writes into has several writers, so the +//! locked, atomic writers for it live here too ([`merge_json_state_file`], +//! [`write_json_state_file`]). +//! //! The loopback HTTP and TCP primitives these probes are built on live in //! `net.rs`; the state-file keys and the probe retry interval that tie the //! probes to a [`ManagedServiceRecord`] live in `service_record.rs`. use anyhow::{Context, Result}; use std::fs; -use std::path::Path; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; use std::time::Duration; +use crate::atomic_write::write_file_atomically; use crate::model_catalog::builtin_model_recipes; use crate::net::{http_get_text_with_auth, http_post_json_with_auth}; +use crate::process::FileLock; use crate::service_record::{ INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, INFERENCE_VERIFIED_STATE_KEY, ManagedServiceRecord, @@ -35,34 +42,190 @@ use crate::unix_time_millis; /// processing before it answers. pub const INFERENCE_PROBE_TIMEOUT: Duration = Duration::from_secs(8); +/// Name of the lock file that serializes every write of a JSON state file in +/// one directory, taken by [`merge_json_state_file`] and +/// [`write_json_state_file`]. +/// +/// One lock per directory rather than one per state file, so it can be left in +/// place for good: deleting a lock file while another writer holds or waits on +/// it would let a third writer lock a fresh file of the same name and run +/// alongside them. It is never removed, holds no data, and its `.lock` +/// extension keeps it out of every scan that looks for `*.json` state files. +pub const JSON_STATE_LOCK_FILE_NAME: &str = ".json-state.lock"; + +/// The lock [`merge_json_state_file`] and [`write_json_state_file`] take for +/// `path`: [`JSON_STATE_LOCK_FILE_NAME`] in `path`'s directory. +#[must_use] +pub fn json_state_lock_path(path: &Path) -> PathBuf { + path.parent() + .unwrap_or_else(|| Path::new("")) + .join(JSON_STATE_LOCK_FILE_NAME) +} + /// Merge `patch`'s top-level keys into the JSON object stored at `path`. /// -/// Creates the file (and its parent) when absent, and replaces a non-object -/// document rather than failing — the caller is recording a fact about a live -/// service, not validating an existing file. +/// Several writers patch the same document — the engine wrapper that owns the +/// service, a healthcheck latching that inference was verified, `rocmd` +/// publishing the Anthropic endpoint — so the read, the merge and the write +/// run under one cross-process lock ([`json_state_lock_path`]) and no merge +/// can drop a key another merge recorded. The lock orders writers rather than +/// combining them: a full rewrite through [`write_json_state_file`] still +/// replaces every key merged before it, and a writer that bypasses both +/// functions (the vLLM and AIM engines' own state writes) takes no lock at all. +/// +/// The result is published atomically +/// ([`crate::atomic_write::write_file_atomically`]), so a reader that takes no +/// lock still sees a whole document, never a truncated one. +/// +/// Creates the file (and its parent) when absent. An existing file that is +/// empty, is not valid JSON or does not hold a JSON object is an error, and the +/// file is left exactly as it was: it may hold the only record of a running +/// process (`server_pid`), and replacing it with `patch` alone would lose it. +/// A `patch` that is not an object has no keys to merge; the file is not +/// touched and the call succeeds. pub fn merge_json_state_file(path: &Path, patch: &serde_json::Value) -> Result<()> { - let mut value = fs::read(path) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .unwrap_or_else(|| serde_json::json!({})); - if !value.is_object() { - value = serde_json::json!({}); - } - let object = value.as_object_mut().expect("object checked above"); - if let Some(patch) = patch.as_object() { - for (key, patch_value) in patch { - object.insert(key.clone(), patch_value.clone()); + merge_json_state_file_with(path, patch, |path| fs::read(path), || {}) +} + +/// [`merge_json_state_file`], reading the existing file through `read` and +/// running `before_publish` under the lock just before the merged document is +/// published, so a test can observe the lock held at both moments. +fn merge_json_state_file_with( + path: &Path, + patch: &serde_json::Value, + read: impl FnOnce(&Path) -> std::io::Result>, + before_publish: impl FnOnce(), +) -> Result<()> { + let Some(patch) = patch.as_object() else { + return Ok(()); + }; + let _lock = FileLock::acquire(json_state_lock_path(path))?; + let mut object = match read(path) { + Ok(bytes) => existing_state_object(path, &bytes)?, + Err(error) if error.kind() == ErrorKind::NotFound => serde_json::Map::new(), + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read {}; left it untouched and recorded nothing", + path.display() + ) + }); } + }; + for (key, patch_value) in patch { + object.insert(key.clone(), patch_value.clone()); + } + before_publish(); + publish_json_state(path, &serde_json::Value::Object(object)) +} + +/// Replace the JSON state file at `path` with `value`. +/// +/// Takes the same lock as [`merge_json_state_file`] and publishes the same +/// atomic way, so a full rewrite cannot land in the middle of another +/// writer's merge. +pub fn write_json_state_file(path: &Path, value: &serde_json::Value) -> Result<()> { + write_json_state_file_with(path, value, || {}) +} + +/// [`write_json_state_file`], running `before_publish` under the lock just +/// before the document is published, so a test can observe the lock held at +/// that moment. +fn write_json_state_file_with( + path: &Path, + value: &serde_json::Value, + before_publish: impl FnOnce(), +) -> Result<()> { + let _lock = FileLock::acquire(json_state_lock_path(path))?; + before_publish(); + publish_json_state(path, value) +} + +/// Why [`merge_json_state_file`] refused an existing state file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JsonStateRefusal { + /// The file holds no bytes. + Empty, + /// The bytes do not parse as JSON. + NotJson, + /// The bytes are a whole JSON document, but not an object. + NotAnObject, +} + +impl JsonStateRefusal { + /// Whether a writer that rewrites the file in place (truncate, then + /// write) could have left it this way for a moment: empty, or a document + /// cut short. A whole document that is not an object is not that. + pub const fn may_be_mid_write(self) -> bool { + matches!(self, Self::Empty | Self::NotJson) } - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; +} + +/// The error [`merge_json_state_file`] returns for a state file it refused +/// and left untouched; reach it with `anyhow::Error::downcast_ref`. +#[derive(Debug)] +pub struct JsonStateFileRefused { + /// Why the file was refused. + pub refusal: JsonStateRefusal, + message: String, +} + +impl std::fmt::Display for JsonStateFileRefused { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) } - fs::write( - path, - serde_json::to_vec_pretty(&value).context("failed to serialize service state")?, - ) - .with_context(|| format!("failed to write {}", path.display())) +} + +impl std::error::Error for JsonStateFileRefused {} + +/// The object an existing state file holds, or a [`JsonStateFileRefused`] +/// that says the file was left untouched. +fn existing_state_object( + path: &Path, + bytes: &[u8], +) -> Result> { + let refuse = |refusal: JsonStateRefusal, what: String| { + anyhow::Error::new(JsonStateFileRefused { + refusal, + message: format!( + "{} {what}; left it untouched and recorded nothing, rather than replace what it \ + holds", + path.display() + ), + }) + }; + if bytes.is_empty() { + return Err(refuse(JsonStateRefusal::Empty, "is empty".to_owned())); + } + match serde_json::from_slice::(bytes) { + Ok(serde_json::Value::Object(object)) => Ok(object), + Ok(other) => Err(refuse( + JsonStateRefusal::NotAnObject, + format!("holds a JSON {}, not an object", json_kind(&other)), + )), + Err(error) => Err(refuse( + JsonStateRefusal::NotJson, + format!("is not valid JSON ({error})"), + )), + } +} + +const fn json_kind(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "boolean", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + +/// Atomic publish of `value`; the caller holds [`json_state_lock_path`]. +fn publish_json_state(path: &Path, value: &serde_json::Value) -> Result<()> { + let bytes = serde_json::to_vec_pretty(value).context("failed to serialize service state")?; + write_file_atomically(path, &bytes) + .with_context(|| format!("failed to write {}", path.display())) } /// Whether inference has been confirmed for a service, from its engine state @@ -76,6 +239,18 @@ pub fn merge_json_state_file(path: &Path, patch: &serde_json::Value) -> Result<( /// The attempt is recorded before the probe runs, so a caller killed mid-probe /// still leaves the throttle in place instead of freeing the next poll to spend /// another full timeout. +/// +/// A call that finds the latch already set returns `true`, and a call with an +/// empty `model_ref` returns `false`; neither probes nor writes. +/// +/// When the attempt cannot be recorded, because the merge refuses the state +/// file (empty, torn or not a JSON object) or fails to lock, read or write it, +/// the file is left as it is and neither the attempt nor a verified probe is +/// recorded: the throttle does not hold, and every call that reaches the probe +/// sends one, spending up to [`INFERENCE_PROBE_TIMEOUT`], for as long as the +/// file stays that way, which may be for good. The failure is not reported, +/// since this crate has no log to report it to and the caller's answer would +/// not change; [`merge_json_state_file`]'s error says why. pub fn engine_state_inference_verified( state_path: &Path, state: Option<&serde_json::Value>, @@ -1004,4 +1179,568 @@ mod tests { ); Ok(()) } + + // The locked, atomic state-file writers. One test per state the file can + // be found in when a merge starts: absent, empty, torn, valid but not an + // object, unreadable, a valid object; plus a patch with nothing to merge. + + fn json_state_dir(tag: &str) -> std::path::PathBuf { + let dir = crate::test_support::workspace_test_artifact_dir().join(format!( + "json-state-{tag}-{}", + crate::test_support::unique_suffix() + )); + fs::create_dir_all(&dir).expect("create the test directory"); + dir + } + + /// The merge refused `path`, said so naming the file and saying it was left + /// untouched, and the file does still hold exactly `before` — the message + /// and the state it claims, checked together. + fn assert_refused_and_untouched( + result: Result<()>, + path: &Path, + before: &[u8], + why: &str, + refusal: JsonStateRefusal, + ) { + let error = result.expect_err("an unusable state file must be refused"); + assert_eq!( + error + .downcast_ref::() + .map(|refused| refused.refusal), + Some(refusal), + "the error is a typed refusal of the right kind: {error:#}" + ); + let error = format!("{error:#}"); + assert!( + error.contains(&path.display().to_string()), + "the error names the file: {error}" + ); + assert!( + error.contains("left it untouched and recorded nothing"), + "the error says the file was left as it was: {error}" + ); + assert!(error.contains(why), "the error says why ({why}): {error}"); + assert_eq!( + fs::read(path).expect("the refused file is still there"), + before, + "a refused merge must leave the file byte for byte as it was" + ); + } + + #[test] + fn merging_into_an_absent_state_file_creates_it_with_the_patch() -> Result<()> { + let dir = json_state_dir("absent"); + let path = dir.join("not-yet").join("svc.json"); + + merge_json_state_file( + &path, + &serde_json::json!({ "status": "starting", "pid": 7 }), + )?; + + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!( + written, + serde_json::json!({ "status": "starting", "pid": 7 }) + ); + Ok(()) + } + + #[test] + fn merging_into_an_empty_state_file_is_refused_and_leaves_it_untouched() { + let dir = json_state_dir("empty"); + let path = dir.join("svc.json"); + fs::write(&path, b"").unwrap(); + + let result = merge_json_state_file(&path, &serde_json::json!({ "status": "ready" })); + + assert_refused_and_untouched(result, &path, b"", "is empty", JsonStateRefusal::Empty); + let _ = fs::remove_dir_all(&dir); + } + + /// The case behind the lost `server_pid`: a writer caught another + /// mid-write, read half a document, and replaced it with its own patch. + #[test] + fn merging_into_a_torn_state_file_is_refused_and_leaves_it_untouched() { + let dir = json_state_dir("torn"); + let path = dir.join("svc.json"); + let torn = br#"{"status": "running", "server_pid": 42"#; + fs::write(&path, torn).unwrap(); + + let result = merge_json_state_file( + &path, + &serde_json::json!({ INFERENCE_PROBE_ATTEMPTED_STATE_KEY: 1 }), + ); + + assert_refused_and_untouched( + result, + &path, + torn, + "is not valid JSON", + JsonStateRefusal::NotJson, + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn merging_into_a_state_file_that_is_not_an_object_is_refused_and_leaves_it_untouched() { + let dir = json_state_dir("non-object"); + let path = dir.join("svc.json"); + for (document, kind) in [ + (&b"[1, 2]"[..], "holds a JSON array, not an object"), + (&b"null"[..], "holds a JSON null, not an object"), + (&b"\"x\""[..], "holds a JSON string, not an object"), + ] { + fs::write(&path, document).unwrap(); + let result = merge_json_state_file(&path, &serde_json::json!({ "status": "ready" })); + assert_refused_and_untouched( + result, + &path, + document, + kind, + JsonStateRefusal::NotAnObject, + ); + } + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn a_state_file_that_cannot_be_read_is_refused_and_left_in_place() { + let dir = json_state_dir("unreadable"); + // A directory where the file should be: reading it fails with an + // error other than "not found" on every platform. + let path = dir.join("svc.json"); + fs::create_dir_all(&path).unwrap(); + + let error = format!( + "{:#}", + merge_json_state_file(&path, &serde_json::json!({ "status": "ready" })) + .expect_err("an unreadable state file must be refused") + ); + + let still_a_directory = path.is_dir(); + let _ = fs::remove_dir_all(&dir); + assert!( + error.contains(&format!("failed to read {}", path.display())), + "{error}" + ); + assert!( + error.contains("left it untouched and recorded nothing"), + "{error}" + ); + assert!(still_a_directory, "the refused path must be left in place"); + } + + #[test] + fn merging_into_a_state_object_keeps_its_other_keys() -> Result<()> { + let dir = json_state_dir("merge"); + let path = dir.join("svc.json"); + fs::write( + &path, + br#"{"status": "running", "server_pid": 42, "server_start_ticks": 9}"#, + )?; + + merge_json_state_file( + &path, + &serde_json::json!({ "status": "ready", INFERENCE_VERIFIED_STATE_KEY: 5 }), + )?; + + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!( + written, + serde_json::json!({ + "status": "ready", + "server_pid": 42, + "server_start_ticks": 9, + INFERENCE_VERIFIED_STATE_KEY: 5, + }) + ); + Ok(()) + } + + #[test] + fn a_patch_that_is_not_an_object_changes_nothing() -> Result<()> { + let dir = json_state_dir("non-object-patch"); + let path = dir.join("svc.json"); + let before = br#"{"server_pid": 42}"#; + fs::write(&path, before)?; + let absent = dir.join("absent.json"); + + merge_json_state_file(&path, &serde_json::json!(["status"]))?; + merge_json_state_file(&absent, &serde_json::Value::Null)?; + + let after = fs::read(&path)?; + let created = absent.exists(); + let _ = fs::remove_dir_all(&dir); + assert_eq!(after, before, "nothing to merge, so nothing is written"); + assert!(!created, "nothing to merge, so no file is created"); + Ok(()) + } + + #[test] + fn writing_a_state_file_replaces_it_whole() -> Result<()> { + let dir = json_state_dir("write"); + let path = dir.join("nested").join("svc.json"); + + write_json_state_file( + &path, + &serde_json::json!({ "status": "starting", "pid": 1 }), + )?; + let first: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + write_json_state_file(&path, &serde_json::json!({ "status": "running" }))?; + let second: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + + let _ = fs::remove_dir_all(&dir); + assert_eq!(first, serde_json::json!({ "status": "starting", "pid": 1 })); + assert_eq!(second, serde_json::json!({ "status": "running" })); + Ok(()) + } + + /// The writers leave behind the state file and the directory's one lock + /// file, and nothing else: no temp file, no per-file lock. + #[test] + fn state_writes_leave_only_the_state_file_and_the_directory_lock() -> Result<()> { + let dir = json_state_dir("leftovers"); + let path = dir.join("svc.json"); + + write_json_state_file(&path, &serde_json::json!({ "status": "starting" }))?; + merge_json_state_file(&path, &serde_json::json!({ "status": "ready" }))?; + let _ = merge_json_state_file(&dir.join("other.json"), &serde_json::json!({ "a": 1 })); + + let mut names: Vec = fs::read_dir(&dir)? + .map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.sort(); + let lock = json_state_lock_path(&path); + let _ = fs::remove_dir_all(&dir); + assert_eq!(names, [JSON_STATE_LOCK_FILE_NAME, "other.json", "svc.json"]); + assert_eq!(lock, dir.join(JSON_STATE_LOCK_FILE_NAME)); + assert_eq!( + lock.extension().and_then(|value| value.to_str()), + Some("lock"), + "scans for `*.json` state files must not pick the lock up" + ); + Ok(()) + } + + /// Another writer's file sitting on the temp name a clock-and-pid writer + /// would pick is neither written through nor moved by a merge. This pins + /// only that; a merge that wrote the state file in place would pass it + /// too, and the torn-read check in + /// `concurrent_merges_across_threads_and_processes_keep_every_key` is what + /// catches that. + #[test] + fn a_merge_spares_another_writers_staged_file() -> Result<()> { + let dir = json_state_dir("foreign-temp"); + let path = dir.join("svc.json"); + crate::foreign_temp_probe::assert_write_spares_foreign_temp_files( + |millis| dir.join(format!("svc.json.tmp-{}-{millis}-0", std::process::id())), + || { + fs::write(&path, br#"{"server_pid": 42}"#)?; + merge_json_state_file(&path, &serde_json::json!({ "status": "ready" })) + }, + )?; + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!( + written, + serde_json::json!({ "server_pid": 42, "status": "ready" }) + ); + Ok(()) + } + + /// Whether one call of [`engine_state_inference_verified`] sends a probe, + /// against an endpoint that answers it as still loading. The endpoint is + /// unblocked afterwards with a connection of its own, so a call that sends + /// nothing is seen as such instead of leaving the endpoint waiting. + fn one_call_probes(path: &Path) -> Result<(bool, bool)> { + let (port, server) = spawn_canned_http_server(vec![Some(( + "HTTP/1.1 503 Service Unavailable", + r#"{"error":"loading model"}"#, + ))])?; + let endpoint = format!("http://127.0.0.1:{port}/v1"); + // As the engine adapters do: the state as read, or none for a file + // that does not parse. + let state = fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let verified = + engine_state_inference_verified(path, state.as_ref(), &endpoint, "qwen", None); + let _ = std::net::TcpStream::connect(("127.0.0.1", port)); + let requests = server.join().expect("server thread should not panic")?; + let probed = requests + .first() + .is_some_and(|request| request.starts_with("POST ")); + Ok((verified, probed)) + } + + /// A refused state file keeps the throttle from being recorded: every call + /// probes the endpoint again, and the file is left as it was. + #[test] + fn a_refused_state_file_cannot_hold_the_probe_throttle() -> Result<()> { + let dir = json_state_dir("refused-throttle"); + let path = dir.join("svc.json"); + let torn = br#"{"status": "ready", "server_pid": 42"#; + fs::write(&path, torn)?; + + let first = one_call_probes(&path)?; + let second = one_call_probes(&path)?; + + let after = fs::read(&path)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!( + first, + (false, true), + "the first call probes a loading engine" + ); + assert_eq!( + second, + (false, true), + "without a recorded attempt the second call probes again" + ); + assert_eq!(after, torn, "the refused file is left as it was"); + Ok(()) + } + + /// The counterpart on a whole state file: the call records its attempt, + /// which is what holds the next call back for + /// [`INFERENCE_PROBE_RETRY_INTERVAL`]. (Whether a second call is then held + /// back depends on how long the first took, so it is not asserted here.) + #[test] + fn a_whole_state_file_records_the_probe_attempt() -> Result<()> { + let dir = json_state_dir("whole-throttle"); + let path = dir.join("svc.json"); + fs::write(&path, br#"{"status": "ready", "server_pid": 42}"#)?; + let before = unix_time_millis() as u64; + + let first = one_call_probes(&path)?; + + let state: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!(first, (false, true), "the call probes a loading engine"); + let attempted = state[INFERENCE_PROBE_ATTEMPTED_STATE_KEY] + .as_u64() + .unwrap_or_else(|| panic!("the attempt is recorded: {state}")); + assert!(attempted >= before, "{state}"); + assert_eq!(state["server_pid"], 42, "{state}"); + Ok(()) + } + + /// Whether the state directory's lock for `path` is held by someone else + /// right now: a non-blocking attempt to take it fails exactly when it is. + fn state_lock_is_held(path: &Path) -> bool { + FileLock::try_acquire(json_state_lock_path(path)) + .expect("try the state directory's lock") + .is_none() + } + + /// A full rewrite publishes while holding the lock every merge holds + /// across its read-modify-write, so it can never land between a merge's + /// read and its write and then be overwritten by the merge's stale copy. + /// Observed at the moment of publishing, so the result does not depend on + /// how fast the host is. + #[test] + fn a_state_file_rewrite_publishes_under_the_directory_lock() -> Result<()> { + let dir = json_state_dir("rewrite-lock"); + let path = dir.join("svc.json"); + let mut held = None; + + write_json_state_file_with(&path, &serde_json::json!({ "status": "starting" }), || { + held = Some(state_lock_is_held(&path)); + })?; + + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let released = !state_lock_is_held(&path); + let _ = fs::remove_dir_all(&dir); + assert_eq!(held, Some(true), "the rewrite must publish under the lock"); + assert!(released, "the lock is released once the rewrite returns"); + assert_eq!(written, serde_json::json!({ "status": "starting" })); + Ok(()) + } + + #[test] + fn a_state_file_merge_publishes_under_the_directory_lock() -> Result<()> { + let dir = json_state_dir("merge-lock"); + let path = dir.join("svc.json"); + fs::write(&path, br#"{"server_pid": 42}"#)?; + let mut held = None; + + merge_json_state_file_with( + &path, + &serde_json::json!({ "status": "ready" }), + |path| fs::read(path), + || { + held = Some(state_lock_is_held(&path)); + }, + )?; + + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let released = !state_lock_is_held(&path); + let _ = fs::remove_dir_all(&dir); + assert_eq!(held, Some(true), "the merge must publish under the lock"); + assert!(released, "the lock is released once the merge returns"); + assert_eq!( + written, + serde_json::json!({ "server_pid": 42, "status": "ready" }) + ); + Ok(()) + } + + /// A merge reads the file it is about to replace while holding the lock, + /// so a write that lands between its read and its publish has to wait for + /// it rather than be overwritten by the merge's stale copy. Observed at the + /// moment of reading, so the result does not depend on how fast the host + /// is. + #[test] + fn a_state_file_merge_reads_under_the_directory_lock() -> Result<()> { + let dir = json_state_dir("merge-read-lock"); + let path = dir.join("svc.json"); + fs::write(&path, br#"{"server_pid": 42}"#)?; + let mut held_at_read = None; + + merge_json_state_file_with( + &path, + &serde_json::json!({ "status": "ready" }), + |read_path| { + held_at_read = Some(state_lock_is_held(read_path)); + fs::read(read_path) + }, + || {}, + )?; + + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + assert_eq!( + held_at_read, + Some(true), + "the merge must read the file under the lock" + ); + assert_eq!( + written, + serde_json::json!({ "server_pid": 42, "status": "ready" }) + ); + Ok(()) + } + + const MERGE_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_JSON_STATE_MERGE_CHILD_PATH"; + const MERGE_CHILD_WRITER_ENV: &str = "ROCM_CORE_TEST_JSON_STATE_MERGE_CHILD_WRITER"; + const MERGES_PER_WRITER: usize = 40; + + fn merge_distinct_keys(path: &Path, writer: &str) -> Result<()> { + for n in 0..MERGES_PER_WRITER { + merge_json_state_file(path, &serde_json::json!({ format!("{writer}-{n}"): n }))?; + } + Ok(()) + } + + /// The regression for the lost `server_pid`: writers in this process and + /// in two child processes (this test binary run again) each merge their + /// own keys into one state file while a reader that takes no lock reads it + /// over and over. Every key and the original `server_pid` must survive, + /// and the reader must only ever see a whole object. + #[test] + fn concurrent_merges_across_threads_and_processes_keep_every_key() -> Result<()> { + use std::sync::atomic::{AtomicBool, Ordering}; + + if let (Some(path), Some(writer)) = ( + std::env::var_os(MERGE_CHILD_PATH_ENV), + std::env::var(MERGE_CHILD_WRITER_ENV).ok(), + ) { + return merge_distinct_keys(Path::new(&path), &writer); + } + + let dir = json_state_dir("concurrent"); + let path = dir.join("svc.json"); + fs::write(&path, br#"{"server_pid": 4242, "server_start_ticks": 7}"#)?; + + let exe = std::env::current_exe().context("current test executable")?; + let child_writers = ["process-a", "process-b"]; + let mut children = Vec::new(); + for writer in child_writers { + children.push( + std::process::Command::new(&exe) + .args([ + "endpoint_readiness::tests::concurrent_merges_across_threads_and_processes_keep_every_key", + "--exact", + "--quiet", + ]) + .env(MERGE_CHILD_PATH_ENV, &path) + .env(MERGE_CHILD_WRITER_ENV, writer) + .stdout(std::process::Stdio::null()) + .spawn() + .context("spawn a merging child process")?, + ); + } + + let done = AtomicBool::new(false); + let thread_writers = ["thread-a", "thread-b", "thread-c", "thread-d"]; + let (thread_results, torn_reads) = std::thread::scope(|scope| { + let reader = scope.spawn(|| { + let mut torn = Vec::new(); + while !done.load(Ordering::Relaxed) { + // Windows may refuse a read while a rename is in flight; a + // refusal is not a torn document, so only what was read + // is judged. + let Ok(bytes) = fs::read(&path) else { continue }; + let whole = serde_json::from_slice::(&bytes) + .ok() + .is_some_and(|value| value.get("server_pid").is_some()); + if !whole { + torn.push(String::from_utf8_lossy(&bytes).into_owned()); + } + } + torn + }); + // Every writer is started before any is joined, so they overlap. + let shared_path = path.as_path(); + let mut writers = Vec::new(); + for writer in thread_writers { + writers.push(scope.spawn(move || merge_distinct_keys(shared_path, writer))); + } + let results: Vec> = writers + .into_iter() + .map(|writer| writer.join().expect("writer thread")) + .collect(); + for child in &mut children { + let _ = child.wait(); + } + done.store(true, Ordering::Relaxed); + (results, reader.join().expect("reader thread")) + }); + + let statuses: Vec<_> = children + .iter_mut() + .map(|child| child.wait().map(|status| status.success())) + .collect::>()?; + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path)?)?; + let _ = fs::remove_dir_all(&dir); + + for result in thread_results { + result?; + } + assert_eq!(statuses, [true, true], "both merging children must succeed"); + assert!( + torn_reads.is_empty(), + "a lock-free reader saw {} partial documents, e.g. {:?}", + torn_reads.len(), + torn_reads.first() + ); + assert_eq!(written["server_pid"], 4242, "{written}"); + assert_eq!(written["server_start_ticks"], 7, "{written}"); + let missing: Vec = thread_writers + .iter() + .chain(child_writers.iter()) + .flat_map(|writer| (0..MERGES_PER_WRITER).map(move |n| format!("{writer}-{n}"))) + .filter(|key| written.get(key).is_none()) + .collect(); + assert!( + missing.is_empty(), + "{} merged keys were lost, e.g. {:?}", + missing.len(), + missing.first() + ); + Ok(()) + } } diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index 98ad844b8d..805bcb9191 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -86,8 +86,8 @@ pub struct Device { /// The oldest distro release the WSL path supports. /// /// Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / `GLIBCXX_3.4.32` floor -/// every published Lemonade embeddable is linked against, so the engine cannot -/// start there. See `docs/wsl.md`. +/// the pinned Lemonade embeddable is linked against (newer Lemonade releases +/// need more), so the engine cannot start there. See `docs/wsl.md`. pub const WSL_MIN_UBUNTU: (u32, u32) = (24, 4); /// WSL2-specific machine state. `None` on every other platform. @@ -124,7 +124,7 @@ pub struct WslFacts { #[serde(default)] pub librocdxg_on_ld_library_path: bool, /// Whether `librocdxg.so` is in the same `lib` directory as a - /// `libhsa-runtime64`, which finds it through its own `$ORIGIN` runpath. A + /// `libhsa-runtime64`, which finds it through its own `$ORIGIN` search path. A /// TheRock tarball is laid out this way and is never in the linker cache. /// Unlike `LD_LIBRARY_PATH` this is a fact about files, so the `--distro` probe /// can ask it from outside the distribution. diff --git a/crates/rocm-core/src/examine/framework.rs b/crates/rocm-core/src/examine/framework.rs index 6817a16a9f..5eb72931da 100644 --- a/crates/rocm-core/src/examine/framework.rs +++ b/crates/rocm-core/src/examine/framework.rs @@ -74,7 +74,35 @@ fn probe_pytorch_in_runtime(e: &mut Examination, interpreter: &FrameworkInterpre "Probed torch with the active managed runtime's interpreter: {}", interpreter.python.display() )); - let env = runtime_library_path_env(e, &interpreter.library_paths); + let library_paths = crate::managed_runtime::runtime_loader_path_entries_read_only( + &interpreter.python, + &interpreter.library_paths, + ); + // The note's claim that a launch or install writes the link is proven by the + // tests of those paths, which live in their own crates: + // `a_launch_carries_the_amd_smi_link_even_when_no_mpi_shim_is_due` (vLLM + // launch), `runtime_environment_carries_the_amd_smi_link_dir` (ComfyUI launch) + // and `the_device_probe_runs_with_the_amd_smi_link_on_its_loader_path` (the + // install device probe). That the note stays away while the link is in place, + // and that the probe never writes it, is + // `the_framework_probe_picks_up_an_existing_amd_smi_link_and_never_writes_one`. + // The note also shows while a leftover stub or `libnuma` link keeps the + // directory off the probe's loader path, since a launch clears those too. That + // clearing is `a_stale_numa_link_is_cleared_by_a_launch_and_held_back_by_a_diagnostic` + // (rocm-core's shared helper, which the ComfyUI launch and the install probe + // reach through `ensure_amd_smi_compat`) and, for the vLLM launch's own call, + // `apply_therock_env_actually_runs_the_compat_shim_step`. + if crate::managed_runtime::amd_smi_link_missing(&interpreter.python, &interpreter.library_paths) + { + e.framework_notes.push( + "The runtime ships libamd_smi.so. without the bare libamd_smi.so name and has no usable \ + amd-smi link yet: a launch or install writes it, and clears any leftover shim beside it \ + that would hide a host library. Until then torch can report 0 devices (seen beside a \ + host ROCm install)." + .to_owned(), + ); + } + let env = runtime_library_path_env(e, &library_paths); let (rc, out, err) = run_with_env( &interpreter.python.display().to_string(), &["-c", PYTORCH_PROBE], @@ -393,6 +421,103 @@ mod tests { }); } + /// The torch probe runs with the amd-smi link's directory on its loader path + /// where an install or launch already made it, and never writes one: an + /// examination must not change the environment it reports on. + #[test] + #[cfg(target_os = "linux")] + fn the_framework_probe_picks_up_an_existing_amd_smi_link_and_never_writes_one() { + use crate::fake_executable::write_fake_executable; + + let _guard = PROCESS_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let root = std::env::temp_dir().join(format!( + "rocm-core-examine-amd-smi-{}", + crate::test_support::unique_suffix() + )); + let sdk_lib = root.join("sdk").join("lib"); + let bin = root.join("env").join("bin"); + let compat = root.join("env").join(crate::RUNTIME_COMPAT_DIR_NAME); + std::fs::create_dir_all(&sdk_lib).unwrap(); + std::fs::create_dir_all(&bin).unwrap(); + std::fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk").unwrap(); + let python = bin.join("python"); + write_fake_executable( + &python, + format!( + "#!/bin/sh\n\ + case \"${env}\" in\n\ + *{compat_name}*) printf '%s' '{ok}' ;;\n\ + *) printf '%s' '{broken}' ;;\n\ + esac\n", + env = RUNTIME_LIBRARY_PATH_ENV, + compat_name = crate::RUNTIME_COMPAT_DIR_NAME, + ok = RUNTIME_TORCH_OK_JSON, + broken = r#"{"ok":false,"error":"no amd-smi link on the loader path"}"#, + ), + ) + .unwrap(); + let interpreter = FrameworkInterpreter { + python, + library_paths: vec![sdk_lib.clone()], + }; + + let mut without = Examination::default(); + probe_framework(&mut without, FrameworkProbe::PyTorch, Some(&interpreter)); + let written = compat.exists(); + + std::fs::create_dir_all(&compat).unwrap(); + std::os::unix::fs::symlink( + sdk_lib.join("libamd_smi.so.26"), + compat.join("libamd_smi.so"), + ) + .unwrap(); + let mut with = Examination::default(); + probe_framework(&mut with, FrameworkProbe::PyTorch, Some(&interpreter)); + + // A leftover `libnuma.so.1` link beside the right link holds the directory + // back (the probe runs without it on its loader path, so the torch import + // still fails), and the note says a launch or install clears it. + std::os::unix::fs::symlink("/does/not/exist", compat.join("libnuma.so.1")).unwrap(); + let mut held = Examination::default(); + probe_framework(&mut held, FrameworkProbe::PyTorch, Some(&interpreter)); + let numa_still_there = compat.join("libnuma.so.1").symlink_metadata().is_ok(); + std::fs::remove_dir_all(&root).ok(); + + assert!(!written, "the probe wrote the shim directory"); + // Without the link the fake interpreter fails with its one specific + // message, so the test goes red only for the missing link directory. + assert!( + without + .framework_notes + .iter() + .any(|note| note == "torch import failed: no amd-smi link on the loader path"), + "{:?}", + without.framework_notes + ); + assert_eq!(without.framework_version, ""); + // A runtime that needs the link and has none says so, since a launch that + // could not write it is silent; with the link there is nothing to say. + let link_note = |e: &Examination| { + e.framework_notes + .iter() + .any(|note| note.contains("has no usable amd-smi link yet")) + }; + assert!(link_note(&without), "{:?}", without.framework_notes); + assert!(!link_note(&with), "{:?}", with.framework_notes); + assert!(numa_still_there, "the probe removed the leftover link"); + assert_eq!(held.framework_version, "", "{:?}", held.framework_notes); + assert!( + held.framework_notes + .iter() + .any(|note| note.contains("clears any leftover shim beside it")), + "{:?}", + held.framework_notes + ); + assert_eq!(with.framework_version, "2.11.0+rocm7.14.1"); + } + #[test] #[cfg(unix)] fn the_framework_probe_reads_the_active_runtimes_torch() { diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 3a6977626d..668e5e2389 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -16,9 +16,11 @@ //! place. //! //! This file is the module root. The exit codes, [`FixOptions`] and [`FixClass`] -//! live in `fix/options.rs`, and the consent decision, the timeouts and the -//! `fail!` helper in `fix/consent.rs`; the names are re-exported from here, so -//! `crate::fix::exit` and `rocm_core::fix::FixClass` keep resolving. The +//! live in `fix/options.rs` and are re-exported from here, so +//! `crate::fix::exit` and `rocm_core::fix::FixClass` keep resolving. +//! `fix/consent.rs` holds the consent decision, the timeouts, the `fail!` +//! helper, and the one way a runner runs a host command and reports how it +//! ended (`run_step`, `Step`, `CheckIn`, `StepFailure`, `read_failure`). The //! runners, one per auto-applicable fix, are in `fix/runners.rs`. The closed //! catalog (`RECIPES`, `FixRecipe`, the lookups by fix-id and the test-only //! oracles) is in `fix/catalog.rs`, its rendering in `fix/render.rs` and its @@ -167,7 +169,8 @@ fn act_on(recipe: &FixRecipe, opts: &FixOptions) -> i32 { Plan::MissingRunner => { // A recipe the catalog says the CLI carries out, with nothing to // carry it out with -> INTERNAL, not FAILED (which is reserved for - // "attempted, and the command failed"). Held by + // "attempted, and a command or the read it planned from failed"). + // Held by // `a_recipe_that_acts_has_a_runner`. fail!("Internal error: a recipe the catalog says is applied here has no runner."); exit::INTERNAL @@ -879,10 +882,12 @@ mod tests { "apply", "act_on", "run_render_group", + "render_group_with", "run_unset_override", "run_unset_override_linux", "report_persistent_override", "run_unset_override_windows", + "unset_override_windows_with", "report_and_clear_override_windows", "run_path_export", "run_path_export_linux", @@ -892,6 +897,7 @@ mod tests { "run_hip_visible_devices_linux", "pin_device_in_rc_file", "run_hip_visible_devices_windows", + "hip_visible_devices_windows", ] { assert!( scan.checked.iter().any(|name| name == runner), diff --git a/crates/rocm-core/src/fix/catalog.rs b/crates/rocm-core/src/fix/catalog.rs index 086a5e58a2..5c4d587a26 100644 --- a/crates/rocm-core/src/fix/catalog.rs +++ b/crates/rocm-core/src/fix/catalog.rs @@ -170,6 +170,15 @@ impl FixRecipe { } } +/// The ids of the recipes whose runners name themselves in a recovery line +/// ("re-run `rocm fix `"). One constant serves the catalog entry and the +/// runner's message, so the command a stopped fix tells the user to re-run is +/// always a recipe that exists. +pub(super) const UNSET_OVERRIDE_ID: &str = "fix-2-unset-override"; +pub(super) const RENDER_GROUP_ID: &str = "fix-4-render-group"; +pub(super) const PATH_ID: &str = "fix-6-path"; +pub(super) const IGPU_DGPU_ID: &str = "fix-9-igpu-dgpu"; + /// The recipe registry. Mirrors the diagnosis catalog; only the small, safe /// fixes carry a `runner`, and what each one does can differ by platform. pub(super) const RECIPES: &[FixRecipe] = &[ @@ -204,7 +213,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ runner: None, }, FixRecipe { - fix_id: "fix-2-unset-override", + fix_id: UNSET_OVERRIDE_ID, title: "Unset HSA_OVERRIDE_GFX_VERSION", rationale: "HSA_OVERRIDE_GFX_VERSION is set, but your GPU now has a native wheel. The override hides the real gfx and causes page faults / OUT_OF_REGISTERS at runtime.", commands: &[ @@ -258,7 +267,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ runner: None, }, FixRecipe { - fix_id: "fix-4-render-group", + fix_id: RENDER_GROUP_ID, title: "Add user to render/video groups", rationale: "The current user can't open /dev/kfd because they aren't in the render group. Adding the user is the safe, standard fix.", commands: &["sudo usermod -a -G render,video \"$USER\""], @@ -296,7 +305,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ runner: None, }, FixRecipe { - fix_id: "fix-6-path", + fix_id: PATH_ID, title: "Add the ROCm/HIP bin directory to PATH", rationale: "Linux: ROCm is installed but its bin directory isn't on PATH, so `rocminfo` / `hipcc` aren't visible to the shell. Windows: the HIP SDK is installed but its bin directory isn't on the User PATH, so `hipInfo.exe` and the runtime DLLs can't be found.", commands: &[ @@ -383,7 +392,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ runner: None, }, FixRecipe { - fix_id: "fix-9-igpu-dgpu", + fix_id: IGPU_DGPU_ID, title: "Hide the iGPU with HIP_VISIBLE_DEVICES", rationale: "Both an APU iGPU and a discrete AMD GPU are visible. Pin the runtime to the dGPU so the iGPU doesn't destabilise it.", commands: &[ @@ -732,7 +741,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ FixRecipe { fix_id: "fix-wsl-5-distro-too-old", title: "Move to a distro release the WSL path supports", - rationale: "Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 floor every published Lemonade embeddable is linked against, so the engine cannot start there at all. This is a hard floor, not a recommendation.", + rationale: "Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 floor the pinned Lemonade embeddable is linked against (newer Lemonade releases need more), so the engine cannot start there at all. This is a hard floor, not a recommendation.", commands: &[ "# From Windows, install a supported distro alongside the current one:", "# wsl --install -d Ubuntu-24.04", diff --git a/crates/rocm-core/src/fix/consent.rs b/crates/rocm-core/src/fix/consent.rs index 95b9d8931f..b6adcadc14 100644 --- a/crates/rocm-core/src/fix/consent.rs +++ b/crates/rocm-core/src/fix/consent.rs @@ -9,9 +9,17 @@ //! selected by (`current_os`), and the stderr/stdout helpers (`fail!`, //! `relay_output`) that cannot themselves panic when a pipe is closed. Used //! by `fix.rs` and its runners. +//! +//! Also the one way a runner runs a host command ([`run_step`]) and reports +//! how it ended ([`Step`], [`StepFailure`]): the line printed and the exit +//! code returned come from the same value, so a timeout cannot be reported as +//! an exit code, nor an unfinished change as "NOT changed". +use super::options::exit; +use crate::command_capture::{CommandCapture, CommandOutcome}; use crate::{runtime_is_linux, runtime_is_windows}; use std::io::{IsTerminal, Write}; +use std::process::Command; use std::time::Duration; pub(super) const RUN_TIMEOUT: Duration = Duration::from_mins(1); @@ -35,6 +43,168 @@ pub(super) fn relay_output(out: &str, err: &str) { let _ = write!(std::io::stderr(), "{err}"); } +/// Run one host command of a fix under `timeout` and say how it ended. +/// +/// The runners take this as an injected +/// `FnMut(&str, &[&str], Duration) -> CommandCapture`, so their tests drive +/// every [`CommandOutcome`] without a real `setx`, `PowerShell` or `sudo`. It +/// calls the bounded executor directly rather than the `probe_exec::run` +/// adapter, which folds a timeout and a failed wait into exit code `124` and +/// a program that cannot start into `127` -- which is how a timed-out +/// `usermod` used to be reported as "usermod exited 124". +/// +/// The executor stops the direct child only (see `process/bounded.rs`), so a +/// command run under `sudo` may still finish after a timeout; [`Step::failure`] +/// says so rather than claiming nothing changed. +pub(super) fn run_step(program: &str, args: &[&str], timeout: Duration) -> CommandCapture { + let mut command = Command::new(program); + command.args(args); + crate::process::run_bounded(command, timeout, program) +} + +/// How a command ended, as the first half of a sentence: "`label` timed out +/// after 60s and was stopped". `timeout` is the bound the command ran under. +fn outcome_phrase(label: &str, outcome: CommandOutcome, timeout: Duration) -> String { + match outcome { + CommandOutcome::Exited(code) => format!("{label} exited {code}"), + CommandOutcome::NotFound => format!("{label} could not be started: it was not found"), + CommandOutcome::SpawnFailed(kind) => format!("{label} could not be started: {kind}"), + CommandOutcome::TimedOut => format!( + "{label} timed out after {}s and was stopped", + timeout.as_secs() + ), + CommandOutcome::WaitFailed => { + format!("{label} could not be waited for and was stopped") + } + CommandOutcome::Signaled(Some(signal)) => { + format!("{label} was killed by signal {signal} before it finished") + } + CommandOutcome::Signaled(None) => format!("{label} was killed before it finished"), + } +} + +/// Whether a command that ended this way may have made its change anyway. +/// +/// A command that ran to its own exit with a failure code, or never started, +/// did not. One that was stopped from outside -- the deadline, a failed wait, +/// a signal -- may have been part-way through, or (under `sudo`) may still be +/// running, so the honest answer is "may or may not". +const fn may_have_acted(outcome: CommandOutcome) -> bool { + matches!( + outcome, + CommandOutcome::TimedOut | CommandOutcome::WaitFailed | CommandOutcome::Signaled(_) + ) +} + +/// One host command a fix runs to make its change, and how to recover when +/// the command is stopped before it reports. +pub(super) struct Step<'a> { + /// The program run. + pub(super) program: &'a str, + /// Its arguments. + pub(super) args: &'a [&'a str], + /// The command as messages name it (`setx`, `PowerShell`, `sudo usermod`). + pub(super) label: &'a str, + /// What the command changes, as the subject of "... NOT changed." + pub(super) what: &'a str, + /// A command the user can run to see whether the change was made. + pub(super) check: &'a str, + /// Where `check` runs, which the line names when it is not a plain shell. + pub(super) check_in: CheckIn, + /// The `rocm fix` invocation that makes the change if it was not made. + pub(super) rerun: &'a str, + /// The bound the command runs under; also the number the timeout line names. + pub(super) timeout: Duration, +} + +/// Where a [`Step`]'s `check` command is run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CheckIn { + /// Any shell: "Check with `id -nG alice`". + Shell, + /// A PowerShell expression, which `cmd` cannot run: "Check in PowerShell + /// with `[Environment]::GetEnvironmentVariable(...)`". + PowerShell, +} + +impl CheckIn { + /// The words before the backticked command. + const fn lead(self) -> &'static str { + match self { + Self::Shell => "Check with", + Self::PowerShell => "Check in PowerShell with", + } + } +} + +/// A failed step: the line to print on stderr and the exit code to return, +/// decided together so the two cannot disagree. +#[derive(Debug, PartialEq, Eq)] +pub(super) struct StepFailure { + pub(super) code: i32, + pub(super) line: String, +} + +impl Step<'_> { + /// `None` when the command exited `0`; otherwise what to print and return. + pub(super) fn failure(&self, outcome: CommandOutcome) -> Option { + if outcome == CommandOutcome::Exited(0) { + return None; + } + let phrase = outcome_phrase(self.label, outcome, self.timeout); + let line = if may_have_acted(outcome) { + format!( + "{phrase}; {} may or may not have changed. {} `{}`, then re-run `{}` if it did not.", + self.what, + self.check_in.lead(), + self.check, + self.rerun + ) + } else { + format!("{phrase}; {} NOT changed.", self.what) + }; + Some(StepFailure { + code: exit::FAILED, + line, + }) + } + + /// Run the command through `run`, relay what it printed, and on failure + /// write the [`StepFailure`] line to `err` and return its code. + pub(super) fn run( + &self, + run: &mut impl FnMut(&str, &[&str], Duration) -> CommandCapture, + err: &mut impl Write, + ) -> Result<(), i32> { + let capture = run(self.program, self.args, self.timeout); + relay_output(&capture.stdout, &capture.stderr); + match self.failure(capture.outcome) { + None => Ok(()), + Some(failure) => { + let _ = writeln!(err, "{}", failure.line); + Err(failure.code) + } + } + } +} + +/// A read that a fix depends on did not complete: nothing has been changed +/// yet, and the fix stops rather than guess what the value was. +pub(super) fn read_failure( + subject: &str, + label: &str, + outcome: CommandOutcome, + timeout: Duration, +) -> StepFailure { + StepFailure { + code: exit::FAILED, + line: format!( + "Could not read {subject}: {}; nothing was changed.", + outcome_phrase(label, outcome, timeout) + ), + } +} + /// The platform family a recipe is selected by. /// /// WSL is its own family, which is why recipes valid on both have to name `wsl` @@ -88,10 +258,9 @@ pub(super) fn confirm(prompt: &str, assume_yes: bool) -> bool { mod tests { use super::*; - /// Restored, not new. This PR made the `wsl` arm of the platform lookup - /// carry more weight, not less -- it now drives the per-platform class, the - /// plan, the recipe printer and the listing -- so the guard tying that arm - /// to the real host detector matters more here than it did before. + /// The `wsl` answer selects the per-platform class, the plan, the recipe + /// printer and the listing, so it must be the real host detector's answer + /// and not a second, drifting one. #[test] fn current_os_reports_wsl_exactly_when_is_wsl_host_does() { // `current_os()`'s wsl branch is `crate::is_wsl_host()`, which is @@ -118,6 +287,154 @@ mod tests { ); } + const STEP: Step<'static> = Step { + program: "setx", + args: &[], + label: "setx", + what: "User-scope X", + check: "check-x", + check_in: CheckIn::Shell, + rerun: "rocm fix fix-x", + timeout: RUN_TIMEOUT, + }; + + /// What [`Step::failure`] reports for `outcome`, written out per variant. + /// The `match` is exhaustive on purpose: a new [`CommandOutcome`] does not + /// compile here until someone decides what a stopped fix says about it. + fn expected(outcome: CommandOutcome) -> Option { + let maybe = "User-scope X may or may not have changed. Check with `check-x`, then \ + re-run `rocm fix fix-x` if it did not."; + let not_changed = "User-scope X NOT changed."; + Some(match outcome { + CommandOutcome::Exited(0) => return None, + CommandOutcome::Exited(_) => format!("setx exited 1; {not_changed}"), + CommandOutcome::NotFound => { + format!("setx could not be started: it was not found; {not_changed}") + } + CommandOutcome::SpawnFailed(_) => { + format!("setx could not be started: permission denied; {not_changed}") + } + CommandOutcome::TimedOut => { + format!("setx timed out after 60s and was stopped; {maybe}") + } + CommandOutcome::WaitFailed => { + format!("setx could not be waited for and was stopped; {maybe}") + } + CommandOutcome::Signaled(Some(_)) => { + format!("setx was killed by signal 9 before it finished; {maybe}") + } + CommandOutcome::Signaled(None) => { + format!("setx was killed before it finished; {maybe}") + } + }) + } + + /// The line and the code come from one value: every outcome but a clean + /// exit is exit 4, a timeout says "timed out" with the bound it ran under, + /// and only a command that ran to its own end (or never started) is + /// reported as having changed nothing. + /// + /// This pins the wording against a stand-in step. What each line claims is + /// proved where a runner prints it, in `fix/runners.rs`: + /// `render_group_child_failure_says_nothing_changed_and_returns_4` and + /// `unset_override_setx_failure_returns_4` (an exit code: NOT changed); + /// `render_group_missing_program_says_it_could_not_start` (never started); + /// `render_group_timeout_says_timed_out_and_how_to_check_not_that_nothing_changed` + /// and `unset_override_setx_timeout_says_timed_out_and_how_to_check` (a + /// timeout: may have changed, how to check, what to re-run); and + /// `every_rerun_id_names_a_recipe_with_a_runner` (the re-run command exists). + /// No runner test drives `WaitFailed` or `Signaled`; they take the + /// timeout's "may or may not have changed" wording from this match. + #[test] + fn step_failure_reports_each_outcome_with_its_own_wording() { + for outcome in [ + CommandOutcome::Exited(0), + CommandOutcome::Exited(1), + CommandOutcome::NotFound, + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + CommandOutcome::TimedOut, + CommandOutcome::WaitFailed, + CommandOutcome::Signaled(Some(9)), + CommandOutcome::Signaled(None), + ] { + let got = STEP.failure(outcome); + match expected(outcome) { + None => assert_eq!(got, None, "{outcome:?} is success"), + Some(line) => assert_eq!( + got, + Some(StepFailure { + code: exit::FAILED, + line + }), + "{outcome:?}" + ), + } + } + } + + /// The runner the production paths use reports a program that is not + /// there as `NotFound`, not as the legacy adapter's `127`, so the line a + /// runner prints for it is "could not be started". + #[test] + fn run_step_reports_a_missing_program_as_not_found() { + let capture = run_step("rocm-fix-no-such-program-345", &[], QUERY_TIMEOUT); + assert_eq!(capture.outcome, CommandOutcome::NotFound); + } + + /// A PowerShell expression is not something `cmd` can run, so the line + /// says where to run it; a plain shell command keeps "Check with". + #[test] + fn a_powershell_check_says_to_run_it_in_powershell() { + let step = Step { + check_in: CheckIn::PowerShell, + ..STEP + }; + let line = step + .failure(CommandOutcome::TimedOut) + .expect("a failure") + .line; + assert!( + line.contains(" Check in PowerShell with `check-x`, then re-run"), + "{line}" + ); + } + + #[test] + fn a_timeout_names_the_bound_the_step_ran_under() { + let step = Step { + timeout: QUERY_TIMEOUT, + ..STEP + }; + let line = step + .failure(CommandOutcome::TimedOut) + .expect("a failure") + .line; + assert!(line.starts_with("setx timed out after 8s "), "{line}"); + } + + /// Pins the wording. That a runner really stops before writing anything + /// after a failed read is proved in `fix/runners.rs` by + /// `pin_device_read_timeout_stops_before_setx`, + /// `unset_override_user_read_timeout_stops_before_anything_is_reported_or_cleared` + /// and `a_read_that_exits_non_zero_stops_every_windows_fix_without_writing`. + #[test] + fn a_failed_read_says_nothing_was_changed() { + assert_eq!( + read_failure( + "the current User PATH", + "PowerShell", + CommandOutcome::Exited(1), + QUERY_TIMEOUT + ), + StepFailure { + code: exit::FAILED, + line: "Could not read the current User PATH: PowerShell exited 1; nothing was \ + changed." + .to_owned() + } + ); + } + #[test] fn yes_flag_approves_without_prompting() { assert_eq!(consent_without_prompt(true, false), Some(true)); diff --git a/crates/rocm-core/src/fix/options.rs b/crates/rocm-core/src/fix/options.rs index a761f19214..d7f199dd5f 100644 --- a/crates/rocm-core/src/fix/options.rs +++ b/crates/rocm-core/src/fix/options.rs @@ -146,18 +146,34 @@ pub mod exit { is a usage error and exits 2 (see `unknown_id`).", }, // Proved in `fix/runners.rs` by - // `a_failed_write_returns_4_rather_than_reporting_success` (a file write) - // and `windows_path_read_failure_aborts_before_plan_or_write` (a failed - // read, under `--dry-run`). A command that exits non-zero (`usermod`, - // `setx`, PowerShell) is not driven by a test: those runners start the - // real program. + // `a_failed_write_returns_4_rather_than_reporting_success` (a file write); + // `render_group_child_failure_says_nothing_changed_and_returns_4`, + // `unset_override_setx_failure_returns_4` and + // `path_export_write_failure_returns_4` (a command that exits non-zero); + // `render_group_timeout_says_timed_out_and_how_to_check_not_that_nothing_changed` + // and `pin_device_setx_timeout_says_timed_out_and_how_to_check` (a + // command that does not finish); and + // `windows_path_read_failure_aborts_before_plan_or_write`, + // `unset_override_user_read_timeout_stops_before_anything_is_reported_or_cleared`, + // `unset_override_machine_read_failure_stops_before_clearing_the_user_scope`, + // `pin_device_read_timeout_stops_before_setx`, + // `unset_override_dry_run_with_a_failed_read_stops_without_a_plan`, + // `unset_override_dry_run_with_a_failed_machine_read_stops_without_a_plan` + // and `pin_device_dry_run_with_a_failed_read_stops_without_a_plan` (a + // failed read, under `--dry-run` too). Those runners take an injected command + // runner, so no test starts the real program. Meaning { name: "failed", code: FAILED, - description: "A step of the fix failed: a command it ran, a file it wrote, or a \ - read of state it needed before planning (the current User PATH \ - on Windows, read even under `--dry-run`). The change that step \ - was to make was not made.", + description: "A step of the fix failed: a command it ran exited non-zero or could \ + not be started, a file it wrote could not be written, or a read of \ + state it needed before planning did not complete (on Windows, the \ + current User value that fix-2, fix-6, and fix-9 with a device index \ + read first, and for fix-2 the Machine value too, even under \ + `--dry-run`). The change that step was to make was not made. If a \ + command instead did not finish (it timed out or was stopped), the \ + change may or may not have been made: stderr says how to check it \ + and which `rocm fix` to re-run.", }, // Proved in `fix/runners.rs` by `declining_consent_returns_5_and_writes_nothing`, // `windows_decline_on_user_scope_returns_5` and @@ -179,7 +195,9 @@ pub mod exit { pub struct FixOptions { /// Skip the interactive confirmation (the user already approved the plan). pub yes: bool, - /// Show the plan without changing anything. + /// Show the plan without changing anything. A Windows fix that reads the + /// current value first stops with [`exit::FAILED`] and no plan when that + /// read fails. pub dry_run: bool, /// For `fix-9-igpu-dgpu`: the discrete GPU index to pin. pub device_index: Option, diff --git a/crates/rocm-core/src/fix/runners.rs b/crates/rocm-core/src/fix/runners.rs index c6c84d2df1..5e11d9d9c2 100644 --- a/crates/rocm-core/src/fix/runners.rs +++ b/crates/rocm-core/src/fix/runners.rs @@ -11,8 +11,12 @@ //! the catalog in `fix/catalog.rs` names the entry point of each recipe's //! runner and the running OS decides which variant runs. -use super::consent::{QUERY_TIMEOUT, RUN_TIMEOUT, confirm, fail, relay_output}; +use super::catalog::{IGPU_DGPU_ID, PATH_ID, RENDER_GROUP_ID, UNSET_OVERRIDE_ID}; +use super::consent::{ + CheckIn, QUERY_TIMEOUT, RUN_TIMEOUT, Step, confirm, fail, read_failure, run_step, +}; use super::options::{FixOptions, exit}; +use crate::command_capture::{CommandCapture, CommandOutcome}; use crate::probe_exec::{run, which}; use crate::quoting::ps_single_quote_escape; #[cfg(test)] @@ -22,6 +26,7 @@ use crate::rocm_install::{ use crate::runtime_is_windows; use std::io::Write; use std::path::{Path, PathBuf}; +use std::time::Duration; // --------------------------------------------------------------------------- // Environment helpers @@ -82,26 +87,31 @@ pub(super) fn run_render_group(opts: &FixOptions) -> i32 { fail!("`sudo` is not on PATH and we are not root; cannot add groups."); return exit::NOT_APPLICABLE; } - let (program, args): (&str, Vec) = if root { - ( - "usermod", - vec![ - "-a".into(), - "-G".into(), - "render,video".into(), - user.clone(), - ], - ) + render_group_with(opts, &user, root, run_step, &mut std::io::stderr()) +} + +/// Does the plan/consent/run work for [`run_render_group`] once the user and +/// the privilege route are known, with the command runner and the stderr sink +/// injected so every way `usermod` can end is testable on any host. +/// +/// A `usermod` stopped by the timeout is reported as such, not as an exit +/// code: the executor stops only `sudo`, so `usermod` may still finish, and +/// the line says how to find out (`id -nG`) and what to re-run. +fn render_group_with( + opts: &FixOptions, + user: &str, + root: bool, + mut run: impl FnMut(&str, &[&str], Duration) -> CommandCapture, + err: &mut impl Write, +) -> i32 { + let usermod_args = ["-a", "-G", "render,video", user]; + let (program, label, args): (&str, &str, Vec<&str>) = if root { + ("usermod", "usermod", usermod_args.to_vec()) } else { ( "sudo", - vec![ - "usermod".into(), - "-a".into(), - "-G".into(), - "render,video".into(), - user.clone(), - ], + "sudo usermod", + std::iter::once("usermod").chain(usermod_args).collect(), ) }; println!("Will run: {program} {}", args.join(" ")); @@ -112,12 +122,20 @@ pub(super) fn run_render_group(opts: &FixOptions) -> i32 { if !confirm("Add user to render,video groups?", opts.yes) { return exit::DECLINED; } - let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); - let (rc, out, err) = run(program, &arg_refs, RUN_TIMEOUT); - relay_output(&out, &err); - if rc != 0 { - fail!("usermod exited {rc}; group membership NOT changed."); - return exit::FAILED; + let check = format!("id -nG {user}"); + let rerun = format!("rocm fix {RENDER_GROUP_ID}"); + let step = Step { + program, + args: &args, + label, + what: "group membership", + check: &check, + check_in: CheckIn::Shell, + rerun: &rerun, + timeout: RUN_TIMEOUT, + }; + if let Err(code) = step.run(&mut run, err) { + return code; } println!("Added {user} to render,video."); println!( @@ -163,8 +181,9 @@ fn run_unset_override_linux() -> i32 { /// Takes the paths rather than deriving them from `$HOME`, for the same reason /// [`pin_device_in_rc_file`] takes its rc path: it makes the promise testable /// without mutating a process-global. And the promise needs testing — fix-2 is -/// flagged `auto_applicable`, so `skills/rocm-doctor/` has to say plainly that -/// this arm still only reports, and something has to hold that true. +/// `FixClass::Auto` on Windows and `FixClass::PrintOnly` here, yet it still +/// has a runner that this arm is part of, so `skills/rocm-doctor/` has to say +/// plainly that this arm only reports, and something has to hold that true. fn report_persistent_override(candidates: &[PathBuf]) -> i32 { let hits: Vec<&PathBuf> = candidates .iter() @@ -193,31 +212,81 @@ fn report_persistent_override(candidates: &[PathBuf]) -> i32 { } fn run_unset_override_windows(opts: &FixOptions) -> i32 { - let current = std::env::var("HSA_OVERRIDE_GFX_VERSION").unwrap_or_default(); + let current = std::env::var(OVERRIDE_VAR).unwrap_or_default(); + unset_override_windows_with( + opts, + ¤t, + run_step, + |prompt| confirm(prompt, opts.yes), + &mut std::io::stdout(), + &mut std::io::stderr(), + ) +} + +/// Reports `current` (this process's own value of the variable), reads both +/// env scopes through `run`, then hands over to +/// [`report_and_clear_override_windows`]. The lines this runner writes itself +/// go to `out` (and its failure lines to `err`), so a test sees them on every +/// path. Two things still reach the real streams: a child's own output, which +/// `Step::run` relays, and everything `confirm` (which production passes in as +/// `consent`) writes -- its `[y/N]` prompt and answer handling on stdout, and +/// its refusal lines ("Non-interactive shell and --yes not passed; refusing to +/// apply.", "Not confirmed; refusing to apply.") on stderr. +/// +/// A read that does not complete stops the fix before anything is printed as +/// found or offered for clearing: read as "unset", it would report "No +/// persistent HSA_OVERRIDE_GFX_VERSION found ... You're done" on a machine +/// that still carries the override, which is the one thing this fix is for. +/// That holds under `--dry-run` too: a plan built from a value that was never +/// read would preview a change the real run might not make. +fn unset_override_windows_with( + opts: &FixOptions, + current: &str, + mut run: impl FnMut(&str, &[&str], Duration) -> CommandCapture, + consent: impl FnOnce(&str) -> bool, + out: &mut impl Write, + err: &mut impl Write, +) -> i32 { if current.is_empty() { - println!("HSA_OVERRIDE_GFX_VERSION is not set in this shell."); + let _ = writeln!(out, "HSA_OVERRIDE_GFX_VERSION is not set in this shell."); } else { - println!("HSA_OVERRIDE_GFX_VERSION={current} is set in this shell."); - println!("Note: clearing it in your Windows env scope does NOT affect this"); - println!("already-open shell -- close and reopen your terminal afterwards."); - } - // A failed read here only affects this report-and-clear path, which never - // overwrites the whole variable (it only clears it via a dedicated - // `setx VAR ""`), so collapsing a failed read to "unset" preserves this - // function's existing behavior; it is `path_export_windows`'s - // read-then-overwrite-the-whole-value shape that made that collapse unsafe. - let user_val = ps_env_scope("HSA_OVERRIDE_GFX_VERSION", "User").unwrap_or_default(); - let machine_val = ps_env_scope("HSA_OVERRIDE_GFX_VERSION", "Machine").unwrap_or_default(); - report_and_clear_override_windows(opts, &user_val, &machine_val, |prompt| { - confirm(prompt, opts.yes) - }) + let _ = writeln!( + out, + "HSA_OVERRIDE_GFX_VERSION={current} is set in this shell." + ); + let _ = writeln!( + out, + "Note: clearing it in your Windows env scope does NOT affect this" + ); + let _ = writeln!( + out, + "already-open shell -- close and reopen your terminal afterwards." + ); + } + let mut values = [String::new(), String::new()]; + for (value, scope) in values.iter_mut().zip(["User", "Machine"]) { + match ps_env_scope(&mut run, OVERRIDE_VAR, scope) { + Ok(read) => *value = read, + Err(failure) => { + let _ = writeln!(err, "{}", failure.line); + return failure.code; + } + } + } + let [user_val, machine_val] = values; + report_and_clear_override_windows(opts, &user_val, &machine_val, consent, run, out, err) } +/// The variable fix-2 clears. +const OVERRIDE_VAR: &str = "HSA_OVERRIDE_GFX_VERSION"; + /// Does the reporting/clearing work for [`run_unset_override_windows`], with -/// the User/Machine scope values and the consent prompt taken as parameters -/// rather than read from `ps_env_scope`/`confirm` directly. That's what makes -/// the decline path testable without a real Windows host -- same reason -/// [`pin_device_in_rc_file`] takes its consent as a closure. +/// the User/Machine scope values, the consent prompt, the command runner and +/// the stdout and stderr sinks taken as parameters rather than read from +/// `ps_env_scope`/`confirm`/`run_step` directly. That's what makes the +/// decline and the `setx` failure paths testable without a real Windows +/// host -- same reason [`pin_device_in_rc_file`] takes its consent as a +/// closure. /// /// A decline must still exit `5`: `skills/rocm-doctor/reference.md` documents /// `5` as "user declined", and a caller (human or agent) relies on that to @@ -232,55 +301,82 @@ fn report_and_clear_override_windows( user_val: &str, machine_val: &str, consent: impl FnOnce(&str) -> bool, + mut run: impl FnMut(&str, &[&str], Duration) -> CommandCapture, + out: &mut impl Write, + err: &mut impl Write, ) -> i32 { if user_val.is_empty() && machine_val.is_empty() { - println!("\nNo persistent HSA_OVERRIDE_GFX_VERSION found in either the User"); - println!("or Machine env scope. You're done after closing/reopening shells."); + let _ = writeln!( + out, + "\nNo persistent HSA_OVERRIDE_GFX_VERSION found in either the User" + ); + let _ = writeln!( + out, + "or Machine env scope. You're done after closing/reopening shells." + ); return exit::OK; } - println!("\nPersistent HSA_OVERRIDE_GFX_VERSION found in:"); + let _ = writeln!(out, "\nPersistent HSA_OVERRIDE_GFX_VERSION found in:"); if !user_val.is_empty() { - println!(" User scope: {user_val}"); + let _ = writeln!(out, " User scope: {user_val}"); } if !machine_val.is_empty() { - println!(" Machine scope: {machine_val}"); + let _ = writeln!(out, " Machine scope: {machine_val}"); } let mut declined = false; if !user_val.is_empty() { - println!("\nClear from the User scope (no admin needed):"); + let _ = writeln!(out, "\nClear from the User scope (no admin needed):"); // What the CLI runs, not a line to paste: the process is started // directly with these arguments, empty one included. Windows PowerShell // 5.1 drops an empty argument, so pasting the line would not clear - // anything; the plan above carries the line to paste. The text is built - // from the argument list that is run, so the two cannot differ. - let setx_args = ["HSA_OVERRIDE_GFX_VERSION", ""]; + // anything; the line to paste is the PowerShell one in the catalog + // entry's `Commands:`, which `act_on` prints before this runner. The + // text is built from the argument list that is run, so the two cannot + // differ. + let setx_args = [OVERRIDE_VAR, ""]; let shown: Vec = setx_args.iter().map(|a| format!("\"{a}\"")).collect(); - println!( + let _ = writeln!( + out, " The CLI runs (arguments, not a line to paste): setx {}", shown.join(" ") ); if opts.dry_run { - println!(" (dry-run; not executed)"); + let _ = writeln!(out, " (dry-run; not executed)"); } else if consent("Clear HSA_OVERRIDE_GFX_VERSION from User scope?") { - let (rc, out, err) = run("setx", &setx_args, RUN_TIMEOUT); - relay_output(&out, &err); - if rc != 0 { - fail!("setx exited {rc}; User scope NOT changed."); - return exit::FAILED; + let check = env_scope_query(OVERRIDE_VAR, "User"); + let rerun = format!("rocm fix {UNSET_OVERRIDE_ID}"); + let step = Step { + program: "setx", + args: &setx_args, + label: "setx", + what: "User-scope HSA_OVERRIDE_GFX_VERSION", + check: &check, + check_in: CheckIn::PowerShell, + rerun: &rerun, + timeout: RUN_TIMEOUT, + }; + if let Err(code) = step.run(&mut run, err) { + return code; } - println!("Cleared from User scope. Reopen your terminal for it to take effect."); + let _ = writeln!( + out, + "Cleared from User scope. Reopen your terminal for it to take effect." + ); } else { declined = true; } } if !machine_val.is_empty() { - println!( + let _ = writeln!( + out, "\nThe Machine scope value cannot be cleared without an Admin shell. Either run an elevated PowerShell and execute:" ); - println!( + let _ = writeln!( + out, " [Environment]::SetEnvironmentVariable('HSA_OVERRIDE_GFX_VERSION', $null, 'Machine')" ); - println!( + let _ = writeln!( + out, "or remove it through System Properties -> Environment Variables -> System variables. This command does NOT elevate itself." ); } @@ -372,7 +468,13 @@ fn run_path_export_windows(opts: &FixOptions) -> i32 { return exit::NOT_APPLICABLE; } let bin_dir = bin_dir.to_string_lossy().into_owned(); - path_export_windows(opts, &bin_dir, || ps_env_scope("PATH", "User")) + path_export_windows( + opts, + &bin_dir, + run_step, + &mut std::io::stdout(), + &mut std::io::stderr(), + ) } /// The literal value that will replace User-scope PATH on a non-empty @@ -396,12 +498,12 @@ fn windows_path_prepend(bin_dir: &str, user_path: &str) -> String { } /// Does the read/plan/write work for [`run_path_export_windows`], with the -/// User-scope PATH read taken as an injected closure rather than calling -/// `ps_env_scope` directly. That's what makes the "read failed" branch -/// testable on any host -- same reason [`report_and_clear_override_windows`] -/// takes consent as a closure. +/// command runner (both the User-scope PATH read and the write) and the stdout +/// and stderr sinks injected. That's what makes the "read failed" and "write stopped" +/// branches testable on any host -- same reason +/// [`report_and_clear_override_windows`] takes consent as a closure. /// -/// A failed read (`None`) must never be treated as "PATH is empty": the +/// A failed read must never be treated as "PATH is empty": the /// write below replaces the *entire* User PATH value, so collapsing a /// PowerShell failure into "nothing there" would wipe the user's real PATH /// down to just `bin_dir`. So a failed read aborts here, before the @@ -411,14 +513,19 @@ fn windows_path_prepend(bin_dir: &str, user_path: &str) -> String { fn path_export_windows( opts: &FixOptions, bin_dir: &str, - read_user_path: impl FnOnce() -> Option, + mut run: impl FnMut(&str, &[&str], Duration) -> CommandCapture, + out: &mut impl Write, + err: &mut impl Write, ) -> i32 { - let Some(user_path) = read_user_path() else { - fail!("Could not read the current User PATH; User PATH NOT changed."); - return exit::FAILED; + let user_path = match ps_env_scope(&mut run, "PATH", "User") { + Ok(user_path) => user_path, + Err(failure) => { + let _ = writeln!(err, "{}", failure.line); + return failure.code; + } }; if !user_path.is_empty() && user_path.to_lowercase().contains(&bin_dir.to_lowercase()) { - println!("User PATH already contains {bin_dir}; no change."); + let _ = writeln!(out, "User PATH already contains {bin_dir}; no change."); return exit::OK; } let new_path = windows_path_prepend(bin_dir, &user_path); @@ -428,10 +535,13 @@ fn path_export_windows( // being shared, above) -- see `ps_single_quote_escape`'s doc comment // for why this escape is required at all. let escaped_new_path = ps_single_quote_escape(&new_path); - println!("Plan: prepend {bin_dir} to your User PATH:"); - println!(" [Environment]::SetEnvironmentVariable('PATH', '{escaped_new_path}', 'User')"); + let _ = writeln!(out, "Plan: prepend {bin_dir} to your User PATH:"); + let _ = writeln!( + out, + " [Environment]::SetEnvironmentVariable('PATH', '{escaped_new_path}', 'User')" + ); if opts.dry_run { - println!("(dry-run; not executed)"); + let _ = writeln!(out, "(dry-run; not executed)"); return exit::OK; } if !confirm("Update User PATH?", opts.yes) { @@ -442,17 +552,23 @@ fn path_export_windows( // normal-length PATH can hit even when the read above succeeded. let script = format!("[Environment]::SetEnvironmentVariable('PATH', '{escaped_new_path}', 'User')"); - let (rc, out, err) = run( - "powershell", - &["-NoProfile", "-Command", &script], - RUN_TIMEOUT, - ); - relay_output(&out, &err); - if rc != 0 { - fail!("PowerShell exited {rc}; User PATH NOT changed."); - return exit::FAILED; + let check = env_scope_query("PATH", "User"); + let rerun = format!("rocm fix {PATH_ID}"); + let step = Step { + program: POWERSHELL, + args: &["-NoProfile", "-Command", &script], + label: "PowerShell", + what: "User PATH", + check: &check, + check_in: CheckIn::PowerShell, + rerun: &rerun, + timeout: RUN_TIMEOUT, + }; + if let Err(code) = step.run(&mut run, err) { + return code; } - println!( + let _ = writeln!( + out, "Added {bin_dir} to your User PATH. This only takes effect in NEW shells -- close this terminal and reopen it before re-running hipInfo." ); exit::OK @@ -552,61 +668,119 @@ fn run_hip_visible_devices_windows(opts: &FixOptions) -> i32 { ); return exit::OK; }; - // Out of scope for ROCMAI-449 (that fix is specific to fix-6-path's - // read-then-overwrite-the-whole-PATH-value shape): a failed read here - // only risks re-running `setx HIP_VISIBLE_DEVICES `, which sets a - // single unrelated variable rather than overwriting PATH. - let existing = ps_env_scope("HIP_VISIBLE_DEVICES", "User").unwrap_or_default(); + hip_visible_devices_windows( + opts, + idx, + run_step, + &mut std::io::stdout(), + &mut std::io::stderr(), + ) +} + +/// Does the read/plan/write work for [`run_hip_visible_devices_windows`] once +/// a device index is known, with the command runner and the stdout and +/// stderr sinks injected, as [`path_export_windows`] does. +/// +/// A read that does not complete stops the fix: read as "unset", it would skip +/// the guard below and overwrite a User-scope value the user set on purpose. +/// Under `--dry-run` it stops as well, rather than preview a `setx` that the +/// guard might refuse once the read works. +fn hip_visible_devices_windows( + opts: &FixOptions, + idx: i64, + mut run: impl FnMut(&str, &[&str], Duration) -> CommandCapture, + out: &mut impl Write, + err: &mut impl Write, +) -> i32 { + let existing = match ps_env_scope(&mut run, "HIP_VISIBLE_DEVICES", "User") { + Ok(existing) => existing, + Err(failure) => { + let _ = writeln!(err, "{}", failure.line); + return failure.code; + } + }; if !existing.is_empty() { - println!( + let _ = writeln!( + out, "User scope already sets HIP_VISIBLE_DEVICES={existing:?}; remove or update it manually rather than overwriting from this command." ); return exit::OK; } - println!("Plan: persist HIP_VISIBLE_DEVICES in the User env scope:"); - println!(" setx HIP_VISIBLE_DEVICES {idx}"); + let _ = writeln!( + out, + "Plan: persist HIP_VISIBLE_DEVICES in the User env scope:" + ); + let _ = writeln!(out, " setx HIP_VISIBLE_DEVICES {idx}"); if opts.dry_run { - println!("(dry-run; not executed)"); + let _ = writeln!(out, "(dry-run; not executed)"); return exit::OK; } if !confirm("Set HIP_VISIBLE_DEVICES in the User scope?", opts.yes) { return exit::DECLINED; } - let (rc, out, err) = run( - "setx", - &["HIP_VISIBLE_DEVICES", &idx.to_string()], - RUN_TIMEOUT, - ); - relay_output(&out, &err); - if rc != 0 { - fail!("setx exited {rc}; HIP_VISIBLE_DEVICES NOT changed."); - return exit::FAILED; + let idx_arg = idx.to_string(); + let check = env_scope_query("HIP_VISIBLE_DEVICES", "User"); + let rerun = format!("rocm fix {IGPU_DGPU_ID} --device-index {idx}"); + let step = Step { + program: "setx", + args: &["HIP_VISIBLE_DEVICES", &idx_arg], + label: "setx", + what: "User-scope HIP_VISIBLE_DEVICES", + check: &check, + check_in: CheckIn::PowerShell, + rerun: &rerun, + timeout: RUN_TIMEOUT, + }; + if let Err(code) = step.run(&mut run, err) { + return code; } - println!( + let _ = writeln!( + out, "setx only takes effect in NEW shells -- close this terminal and reopen it before re-running your workload." ); exit::OK } -/// Read a Windows environment variable from a given scope via PowerShell. +/// The program the Windows runners read and write env scopes with. +const POWERSHELL: &str = "powershell"; + +/// The PowerShell expression that reads `var` from `scope`. Both the read and +/// the "Check in PowerShell with" line of a stopped write use it, so the +/// check a user is told to run is the read this fix itself trusts. +fn env_scope_query(var: &str, scope: &str) -> String { + format!("[Environment]::GetEnvironmentVariable('{var}','{scope}')") +} + +/// Read a Windows environment variable from a given scope via PowerShell, +/// run through `run`. /// -/// Returns `None` if the read itself failed -- PowerShell missing, blocked by -/// execution policy, or the query timing out -- which must stay distinguishable -/// from `Some(String::new())`, meaning the read succeeded and the variable is +/// `Err` if the read itself did not complete -- PowerShell missing, blocked by +/// execution policy, exiting non-zero, or timing out -- carrying the line and +/// exit code to stop the fix with. That must stay distinguishable from +/// `Ok(String::new())`, meaning the read succeeded and the variable is /// genuinely unset/empty in that scope. Collapsing both to `""` is what let a /// failed PATH read look like an empty PATH to a caller that then wrote the -/// "empty" value back out, wiping the real one (ROCMAI-449). -fn ps_env_scope(var: &str, scope: &str) -> Option { - let script = format!("[Environment]::GetEnvironmentVariable('{var}','{scope}')"); - let (rc, out, _) = run( - "powershell", - &["-NoProfile", "-Command", &script], - QUERY_TIMEOUT, - ); - if rc == 0 { - Some(out.trim().to_owned()) - } else { - None +/// "empty" value back out, wiping the real one (ROCMAI-449), and let fix-2 +/// and fix-9 act on a value they never read. +fn ps_env_scope( + run: &mut impl FnMut(&str, &[&str], Duration) -> CommandCapture, + var: &str, + scope: &str, +) -> Result { + // One binding for both the bound the read runs under and the seconds its + // failure line names. `a_read_timeout_names_the_bound_the_read_ran_under` + // pins the bound the read actually runs under. + let timeout = QUERY_TIMEOUT; + let script = env_scope_query(var, scope); + let capture = run(POWERSHELL, &["-NoProfile", "-Command", &script], timeout); + match capture.outcome { + CommandOutcome::Exited(0) => Ok(capture.stdout.trim().to_owned()), + outcome => Err(read_failure( + &format!("the current {scope} {var}"), + "PowerShell", + outcome, + timeout, + )), } } @@ -890,6 +1064,776 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } + // ── Host commands: every way a step's command can end ────────── + // + // ROCMAI-345: the runners used to read the `probe_exec::run` adapter's + // collapsed integer, so a timed-out `usermod`/`setx`/PowerShell was + // reported as "exited 124; ... NOT changed." -- which named no timeout, + // gave no way to recover, and claimed a state nobody had checked (the + // executor stops only the direct child, so `usermod` under `sudo` can + // still finish). And the Windows reads folded a failed read into "unset", + // so fix-2 reported "You're done" and fix-9 skipped its guard and wrote. + // These drive each runner through `FakeHost` and assert the exit code, the + // commands that ran and the stderr line in the same test. + + /// A stand-in for the host commands a runner runs. A PowerShell read + /// (`GetEnvironmentVariable`) answers `read` (or `machine_read` for the + /// Machine scope, when set); every other command ends with `write`. Each + /// call is recorded as its argv and the timeout it was run under. + struct FakeHost<'a> { + read: (CommandOutcome, &'a str), + machine_read: Option<(CommandOutcome, &'a str)>, + write: CommandOutcome, + calls: std::cell::RefCell, Duration)>>, + } + + impl<'a> FakeHost<'a> { + fn new(read: (CommandOutcome, &'a str), write: CommandOutcome) -> Self { + Self { + read, + machine_read: None, + write, + calls: std::cell::RefCell::new(Vec::new()), + } + } + + fn with_machine_read(mut self, machine_read: (CommandOutcome, &'a str)) -> Self { + self.machine_read = Some(machine_read); + self + } + + fn runner(&self) -> impl FnMut(&str, &[&str], Duration) -> CommandCapture + '_ { + move |program, args, timeout| { + let argv: Vec = std::iter::once(program) + .chain(args.iter().copied()) + .map(str::to_owned) + .collect(); + self.calls.borrow_mut().push((argv.clone(), timeout)); + let (outcome, stdout) = if !is_read(&argv) { + (self.write, "") + } else if argv.iter().any(|a| a.contains("'Machine'")) { + self.machine_read.unwrap_or(self.read) + } else { + self.read + }; + CommandCapture::from_run( + argv, + outcome, + None, + stdout.as_bytes().to_vec(), + Vec::new(), + None, + ) + } + } + + fn calls(&self) -> Vec> { + self.calls.borrow().iter().map(|(a, _)| a.clone()).collect() + } + + /// The commands that were not reads, with the timeout each ran under. + fn writes_with_timeouts(&self) -> Vec<(Vec, Duration)> { + self.calls + .borrow() + .iter() + .filter(|(a, _)| !is_read(a)) + .cloned() + .collect() + } + + /// The timeout each read ran under, in call order. + fn read_timeouts(&self) -> Vec { + self.calls + .borrow() + .iter() + .filter(|(a, _)| is_read(a)) + .map(|(_, t)| *t) + .collect() + } + + fn writes(&self) -> Vec> { + self.writes_with_timeouts() + .into_iter() + .map(|(a, _)| a) + .collect() + } + } + + fn is_read(argv: &[String]) -> bool { + argv.iter().any(|a| a.contains("GetEnvironmentVariable")) + } + + const fn exited(code: i32) -> CommandOutcome { + CommandOutcome::Exited(code) + } + + /// A read that succeeded and found the variable unset. + const fn unset() -> (CommandOutcome, &'static str) { + (CommandOutcome::Exited(0), "") + } + + fn approved() -> FixOptions { + FixOptions { + yes: true, + ..FixOptions::default() + } + } + + /// A captured stdout or stderr sink as text. + fn text(captured: &[u8]) -> String { + String::from_utf8(captured.to_vec()).expect("captured output is UTF-8") + } + + // fix-4 ------------------------------------------------------------ + + const SUDO_USERMOD: [&str; 6] = ["sudo", "usermod", "-a", "-G", "render,video", "alice"]; + + fn render_group(write: CommandOutcome) -> (i32, Vec>, Duration, String) { + let host = FakeHost::new(unset(), write); + let mut err = Vec::new(); + let code = render_group_with(&approved(), "alice", false, host.runner(), &mut err); + let timeout = host + .writes_with_timeouts() + .first() + .map_or(Duration::ZERO, |(_, t)| *t); + (code, host.calls(), timeout, text(&err)) + } + + #[test] + fn render_group_success_runs_usermod_under_sudo_and_returns_0() { + let (code, calls, _, err) = render_group(exited(0)); + assert_eq!(code, exit::OK); + assert_eq!(calls, [SUDO_USERMOD.to_vec()]); + assert_eq!(err, "", "a success prints no failure line"); + } + + #[test] + fn render_group_as_root_runs_usermod_directly() { + let host = FakeHost::new(unset(), exited(0)); + let code = render_group_with(&approved(), "alice", true, host.runner(), &mut Vec::new()); + assert_eq!(code, exit::OK); + assert_eq!(host.calls(), [SUDO_USERMOD[1..].to_vec()]); + } + + #[test] + fn render_group_child_failure_says_nothing_changed_and_returns_4() { + let (code, calls, _, err) = render_group(exited(1)); + assert_eq!(code, exit::FAILED); + assert_eq!(calls, [SUDO_USERMOD.to_vec()]); + assert_eq!( + err, + "sudo usermod exited 1; group membership NOT changed.\n" + ); + } + + #[test] + fn render_group_timeout_says_timed_out_and_how_to_check_not_that_nothing_changed() { + let (code, calls, timeout, err) = render_group(CommandOutcome::TimedOut); + assert_eq!(code, exit::FAILED, "a timeout is still a failed attempt"); + assert_eq!(calls, [SUDO_USERMOD.to_vec()]); + assert_eq!( + timeout, RUN_TIMEOUT, + "the line names the bound it ran under" + ); + assert_eq!( + err, + "sudo usermod timed out after 60s and was stopped; group membership may or may not \ + have changed. Check with `id -nG alice`, then re-run `rocm fix fix-4-render-group` \ + if it did not.\n" + ); + } + + #[test] + fn render_group_missing_program_says_it_could_not_start() { + let (code, _, _, err) = render_group(CommandOutcome::NotFound); + assert_eq!(code, exit::FAILED); + assert_eq!( + err, + "sudo usermod could not be started: it was not found; group membership NOT changed.\n" + ); + } + + #[test] + fn render_group_dry_run_runs_nothing() { + let host = FakeHost::new(unset(), exited(0)); + let opts = FixOptions { + dry_run: true, + ..approved() + }; + let code = render_group_with(&opts, "alice", false, host.runner(), &mut Vec::new()); + assert_eq!(code, exit::OK); + assert!(host.calls().is_empty()); + } + + // fix-2 (Windows) -------------------------------------------------- + + /// Run fix-2's Windows arm against `host` with consent granted; returns + /// the exit code, stdout and stderr. + fn unset_override_with(host: &FakeHost<'_>, opts: &FixOptions) -> (i32, String, String) { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = + unset_override_windows_with(opts, "", host.runner(), |_| true, &mut out, &mut err); + (code, text(&out), text(&err)) + } + + fn unset_override(host: &FakeHost<'_>) -> (i32, String) { + let (code, _, err) = unset_override_with(host, &approved()); + (code, err) + } + + /// What fix-2 on Windows prints first when this process does not carry + /// the override (the helpers above pass `""` for it). + const SHELL_UNSET: &str = "HSA_OVERRIDE_GFX_VERSION is not set in this shell.\n"; + + /// What the real command prints on a failed read, with this shell's own + /// value set: the shell-state lines, then nothing about either scope. + #[test] + fn unset_override_failed_read_prints_only_the_shell_state() { + let host = FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = unset_override_windows_with( + &approved(), + "10.3.0", + host.runner(), + |_| panic!("a failed read must not reach the consent gate"), + &mut out, + &mut err, + ); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!( + text(&out), + "HSA_OVERRIDE_GFX_VERSION=10.3.0 is set in this shell.\nNote: clearing it in your \ + Windows env scope does NOT affect this\nalready-open shell -- close and reopen your \ + terminal afterwards.\n" + ); + assert!( + text(&err).starts_with("Could not read the current User HSA_OVERRIDE_GFX_VERSION:"), + "{}", + text(&err) + ); + } + + #[test] + fn unset_override_clears_a_user_scope_value_and_returns_0() { + let host = FakeHost::new((exited(0), "10.3.0\r\n"), exited(0)).with_machine_read(unset()); + let (code, out, err) = unset_override_with(&host, &approved()); + assert_eq!(code, exit::OK); + assert_eq!(host.writes(), [vec!["setx", OVERRIDE_VAR, ""]]); + assert_eq!(err, ""); + assert!( + out.starts_with(&format!( + "{SHELL_UNSET}\nPersistent HSA_OVERRIDE_GFX_VERSION found in:\n User scope: \ + 10.3.0\n" + )), + "{out}" + ); + assert!( + out.ends_with("Cleared from User scope. Reopen your terminal for it to take effect.\n"), + "{out}" + ); + } + + #[test] + fn unset_override_setx_failure_returns_4() { + let host = FakeHost::new((exited(0), "10.3.0"), exited(1)).with_machine_read(unset()); + let (code, out, err) = unset_override_with(&host, &approved()); + assert!( + !out.contains("Cleared"), + "a failed setx is not reported as cleared:\n{out}" + ); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes(), [vec!["setx", OVERRIDE_VAR, ""]]); + assert_eq!( + err, + "setx exited 1; User-scope HSA_OVERRIDE_GFX_VERSION NOT changed.\n" + ); + } + + #[test] + fn unset_override_setx_timeout_says_timed_out_and_how_to_check() { + let host = FakeHost::new((exited(0), "10.3.0"), CommandOutcome::TimedOut) + .with_machine_read(unset()); + let (code, err) = unset_override(&host); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes_with_timeouts()[0].1, RUN_TIMEOUT); + assert_eq!( + err, + "setx timed out after 60s and was stopped; User-scope HSA_OVERRIDE_GFX_VERSION may or \ + may not have changed. Check in PowerShell with \ + `[Environment]::GetEnvironmentVariable('HSA_OVERRIDE_GFX_VERSION','User')`, then \ + re-run `rocm fix fix-2-unset-override` if it did not.\n" + ); + } + + #[test] + fn unset_override_setx_missing_says_it_could_not_start() { + let host = FakeHost::new((exited(0), "10.3.0"), CommandOutcome::NotFound) + .with_machine_read(unset()); + let (code, err) = unset_override(&host); + assert_eq!(code, exit::FAILED); + assert_eq!( + err, + "setx could not be started: it was not found; User-scope HSA_OVERRIDE_GFX_VERSION NOT \ + changed.\n" + ); + } + + /// The reported bug's read half: a User-scope read that timed out used to + /// read as "unset", and with the Machine scope unset too the fix printed + /// "No persistent HSA_OVERRIDE_GFX_VERSION found ... You're done" and + /// exited 0 on a machine that still carried the override. + #[test] + fn unset_override_user_read_timeout_stops_before_anything_is_reported_or_cleared() { + let host = + FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)).with_machine_read(unset()); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = unset_override_windows_with( + &approved(), + "", + host.runner(), + |_| panic!("a failed read must not reach the consent gate"), + &mut out, + &mut err, + ); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty(), "nothing may be written"); + assert_eq!( + text(&out), + SHELL_UNSET, + "after the shell's own value, nothing is reported as found or absent from a read \ + that failed" + ); + assert_eq!( + host.calls().len(), + 1, + "the Machine scope is not read either" + ); + assert_eq!( + text(&err), + "Could not read the current User HSA_OVERRIDE_GFX_VERSION: PowerShell timed out after \ + 8s and was stopped; nothing was changed.\n" + ); + } + + #[test] + fn unset_override_missing_powershell_stops_before_anything_is_cleared() { + let host = FakeHost::new((CommandOutcome::NotFound, ""), exited(0)); + let (code, err) = unset_override(&host); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!( + err, + "Could not read the current User HSA_OVERRIDE_GFX_VERSION: PowerShell could not be \ + started: it was not found; nothing was changed.\n" + ); + } + + /// A Machine-scope read that fails must stop the fix too, even when the + /// User scope has a value to clear: the report would otherwise omit the + /// Machine-scope guidance for a value that may well be there. + #[test] + fn unset_override_machine_read_failure_stops_before_clearing_the_user_scope() { + let host = FakeHost::new((exited(0), "10.3.0"), exited(0)) + .with_machine_read((CommandOutcome::TimedOut, "")); + let (code, err) = unset_override(&host); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty(), "setx must not run"); + assert_eq!( + err, + "Could not read the current Machine HSA_OVERRIDE_GFX_VERSION: PowerShell timed out \ + after 8s and was stopped; nothing was changed.\n" + ); + } + + #[test] + fn unset_override_with_both_scopes_genuinely_unset_is_done_without_writing() { + let host = FakeHost::new(unset(), exited(0)); + let (code, out, err) = unset_override_with(&host, &approved()); + assert_eq!(code, exit::OK); + assert_eq!(host.calls().len(), 2, "both scopes are read"); + assert_eq!( + out, + format!( + "{SHELL_UNSET}\nNo persistent HSA_OVERRIDE_GFX_VERSION found in either the \ + User\nor Machine env scope. You're done after closing/reopening shells.\n" + ) + ); + assert!(host.writes().is_empty()); + assert_eq!(err, ""); + } + + // fix-6 (Windows) -------------------------------------------------- + + /// Run fix-6's Windows arm for `C:\ROCm\bin` against `host`; returns the + /// exit code, stdout and stderr. + fn path_export_with(host: &FakeHost<'_>, opts: &FixOptions) -> (i32, String, String) { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = path_export_windows(opts, r"C:\ROCm\bin", host.runner(), &mut out, &mut err); + (code, text(&out), text(&err)) + } + + fn path_export(host: &FakeHost<'_>) -> (i32, String) { + let (code, _, err) = path_export_with(host, &approved()); + (code, err) + } + + #[test] + fn path_export_success_writes_the_prepended_path_and_returns_0() { + let host = FakeHost::new((exited(0), r"C:\Windows"), exited(0)); + let (code, out, err) = path_export_with(&host, &approved()); + assert_eq!(code, exit::OK); + assert!( + out.ends_with( + "Added C:\\ROCm\\bin to your User PATH. This only takes effect in NEW shells -- \ + close this terminal and reopen it before re-running hipInfo.\n" + ), + "{out}" + ); + let writes = host.writes(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0][..3], ["powershell", "-NoProfile", "-Command"]); + assert!( + writes[0][3].contains(r"'C:\ROCm\bin;C:\Windows'"), + "{:?}", + writes[0] + ); + assert_eq!(err, ""); + } + + #[test] + fn path_export_write_failure_returns_4() { + let host = FakeHost::new((exited(0), r"C:\Windows"), exited(1)); + let (code, err) = path_export(&host); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes().len(), 1); + assert_eq!(err, "PowerShell exited 1; User PATH NOT changed.\n"); + } + + #[test] + fn path_export_write_timeout_says_timed_out_and_how_to_check() { + let host = FakeHost::new((exited(0), r"C:\Windows"), CommandOutcome::TimedOut); + let (code, err) = path_export(&host); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes_with_timeouts()[0].1, RUN_TIMEOUT); + assert_eq!( + err, + "PowerShell timed out after 60s and was stopped; User PATH may or may not have \ + changed. Check in PowerShell with \ + `[Environment]::GetEnvironmentVariable('PATH','User')`, then \ + re-run `rocm fix fix-6-path` if it did not.\n" + ); + } + + #[test] + fn path_export_write_missing_powershell_says_it_could_not_start() { + let host = FakeHost::new((exited(0), r"C:\Windows"), CommandOutcome::NotFound); + let (code, err) = path_export(&host); + assert_eq!(code, exit::FAILED); + assert_eq!( + err, + "PowerShell could not be started: it was not found; User PATH NOT changed.\n" + ); + } + + #[test] + fn path_export_read_timeout_stops_before_the_write_and_says_why() { + let host = FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)); + let (code, err) = path_export(&host); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!( + err, + "Could not read the current User PATH: PowerShell timed out after 8s and was stopped; \ + nothing was changed.\n" + ); + } + + // fix-9 (Windows) -------------------------------------------------- + + /// Run fix-9's Windows arm for device 1 against `host`; returns the exit + /// code, stdout and stderr. + fn pin_device_with(host: &FakeHost<'_>, opts: &FixOptions) -> (i32, String, String) { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = hip_visible_devices_windows(opts, 1, host.runner(), &mut out, &mut err); + (code, text(&out), text(&err)) + } + + fn pin_device(host: &FakeHost<'_>) -> (i32, String) { + let (code, _, err) = pin_device_with(host, &approved()); + (code, err) + } + + #[test] + fn pin_device_success_runs_setx_and_returns_0() { + let host = FakeHost::new(unset(), exited(0)); + let (code, out, err) = pin_device_with(&host, &approved()); + assert_eq!(code, exit::OK); + assert_eq!(host.writes(), [vec!["setx", "HIP_VISIBLE_DEVICES", "1"]]); + assert!( + out.ends_with( + "setx only takes effect in NEW shells -- close this terminal and reopen it \ + before re-running your workload.\n" + ), + "{out}" + ); + assert_eq!(err, ""); + } + + #[test] + fn pin_device_setx_failure_returns_4() { + let host = FakeHost::new(unset(), exited(1)); + let (code, out, err) = pin_device_with(&host, &approved()); + assert!(!out.contains("setx only takes effect"), "{out}"); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes().len(), 1); + assert_eq!( + err, + "setx exited 1; User-scope HIP_VISIBLE_DEVICES NOT changed.\n" + ); + } + + #[test] + fn pin_device_setx_timeout_says_timed_out_and_how_to_check() { + let host = FakeHost::new(unset(), CommandOutcome::TimedOut); + let (code, err) = pin_device(&host); + assert_eq!(code, exit::FAILED); + assert_eq!(host.writes_with_timeouts()[0].1, RUN_TIMEOUT); + assert_eq!( + err, + "setx timed out after 60s and was stopped; User-scope HIP_VISIBLE_DEVICES may or may \ + not have changed. Check in PowerShell with \ + `[Environment]::GetEnvironmentVariable('HIP_VISIBLE_DEVICES','User')`, then re-run \ + `rocm fix fix-9-igpu-dgpu --device-index 1` if it did not.\n" + ); + } + + #[test] + fn pin_device_setx_missing_says_it_could_not_start() { + let host = FakeHost::new(unset(), CommandOutcome::NotFound); + let (code, err) = pin_device(&host); + assert_eq!(code, exit::FAILED); + assert_eq!( + err, + "setx could not be started: it was not found; User-scope HIP_VISIBLE_DEVICES NOT \ + changed.\n" + ); + } + + /// The reported bug's other read: a failed read used to read as "unset", + /// skip the "User scope already sets HIP_VISIBLE_DEVICES" guard and run + /// `setx` over whatever the user had set. + #[test] + fn pin_device_read_timeout_stops_before_setx() { + let host = FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)); + let (code, err) = pin_device(&host); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty(), "setx must not run"); + assert_eq!( + err, + "Could not read the current User HIP_VISIBLE_DEVICES: PowerShell timed out after 8s \ + and was stopped; nothing was changed.\n" + ); + } + + #[test] + fn pin_device_read_with_missing_powershell_stops_before_setx() { + let host = FakeHost::new((CommandOutcome::NotFound, ""), exited(0)); + let (code, _) = pin_device(&host); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty(), "setx must not run"); + } + + #[test] + fn pin_device_leaves_an_existing_user_value_alone() { + let host = FakeHost::new((exited(0), "0"), exited(0)); + let (code, out, _) = pin_device_with(&host, &approved()); + assert_eq!(code, exit::OK); + assert_eq!( + out, + "User scope already sets HIP_VISIBLE_DEVICES=\"0\"; remove or update it manually \ + rather than overwriting from this command.\n" + ); + assert!( + host.writes().is_empty(), + "an existing value is not overwritten" + ); + } + + // Reads that exit non-zero, and the bound a read runs under ----------- + + /// A PowerShell read that runs to its end with a failure code is a failed + /// read, not an answer: each read every Windows fix depends on stops the + /// fix with exit 4, no write, and a line naming the exit code. + #[test] + fn a_read_that_exits_non_zero_stops_every_windows_fix_without_writing() { + let failed = (exited(1), ""); + let cases: [(&str, FakeHost<'_>, &str); 4] = [ + ( + "fix-2 User", + FakeHost::new(failed, exited(0)).with_machine_read(unset()), + "User HSA_OVERRIDE_GFX_VERSION", + ), + ( + "fix-2 Machine", + FakeHost::new((exited(0), "10.3.0"), exited(0)).with_machine_read(failed), + "Machine HSA_OVERRIDE_GFX_VERSION", + ), + ("fix-6", FakeHost::new(failed, exited(0)), "User PATH"), + ( + "fix-9", + FakeHost::new(failed, exited(0)), + "User HIP_VISIBLE_DEVICES", + ), + ]; + for (fix, host, subject) in cases { + let (code, out, err) = match fix { + "fix-6" => path_export_with(&host, &approved()), + "fix-9" => pin_device_with(&host, &approved()), + _ => unset_override_with(&host, &approved()), + }; + assert_eq!(code, exit::FAILED, "{fix}"); + assert!(host.writes().is_empty(), "{fix} must not write"); + let expected = if fix.starts_with("fix-2") { + SHELL_UNSET + } else { + "" + }; + assert_eq!( + out, expected, + "{fix} reports nothing from a read that failed" + ); + assert_eq!( + err, + format!( + "Could not read the current {subject}: PowerShell exited 1; nothing was \ + changed.\n" + ), + "{fix}" + ); + } + } + + /// The bound a read runs under and the seconds its timeout line names are + /// one value: every read runs under `QUERY_TIMEOUT`, and the line names + /// the bound that read actually ran under. + #[test] + fn a_read_timeout_names_the_bound_the_read_ran_under() { + let host = FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)); + let (code, err) = pin_device(&host); + assert_eq!(code, exit::FAILED); + let ran_under = host.read_timeouts(); + assert_eq!(ran_under, [QUERY_TIMEOUT], "one read, under QUERY_TIMEOUT"); + assert!( + err.contains(&format!( + "PowerShell timed out after {}s and was stopped", + ran_under[0].as_secs() + )), + "{err}" + ); + } + + // --dry-run against the Windows reads ----------------------------------- + + fn dry_run() -> FixOptions { + FixOptions { + dry_run: true, + ..approved() + } + } + + /// A dry run with a readable scope prints the plan and changes nothing, + /// even with `--yes`: the read runs, `setx` does not. + #[test] + fn pin_device_dry_run_prints_the_plan_and_runs_no_setx() { + let host = FakeHost::new(unset(), exited(0)); + let (code, out, err) = pin_device_with(&host, &dry_run()); + assert_eq!(code, exit::OK); + assert_eq!(host.calls().len(), 1, "only the read runs"); + assert!(host.writes().is_empty(), "a dry run runs no setx"); + assert_eq!( + out, + "Plan: persist HIP_VISIBLE_DEVICES in the User env scope:\n setx \ + HIP_VISIBLE_DEVICES 1\n(dry-run; not executed)\n" + ); + assert_eq!(err, ""); + } + + /// A dry run whose read fails stops like a real run: exit 4, no plan, no + /// write. A plan built on a value that was never read could preview a + /// `setx` the guard would refuse. + #[test] + fn pin_device_dry_run_with_a_failed_read_stops_without_a_plan() { + let host = FakeHost::new((CommandOutcome::TimedOut, ""), exited(0)); + let (code, out, err) = pin_device_with(&host, &dry_run()); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!(out, "", "no plan is printed"); + assert!( + err.starts_with("Could not read the current User HIP_VISIBLE_DEVICES"), + "{err}" + ); + } + + #[test] + fn unset_override_dry_run_prints_the_plan_and_runs_no_setx() { + let host = FakeHost::new((exited(0), "10.3.0"), exited(0)).with_machine_read(unset()); + let (code, out, err) = unset_override_with(&host, &dry_run()); + assert_eq!(code, exit::OK); + assert_eq!(host.calls().len(), 2, "only the two reads run"); + assert!(host.writes().is_empty(), "a dry run runs no setx"); + assert!( + out.contains( + "The CLI runs (arguments, not a line to paste): setx \ + \"HSA_OVERRIDE_GFX_VERSION\" \"\"\n (dry-run; not executed)\n" + ), + "{out}" + ); + assert_eq!(err, ""); + } + + #[test] + fn unset_override_dry_run_with_a_failed_read_stops_without_a_plan() { + let host = FakeHost::new((CommandOutcome::NotFound, ""), exited(0)); + let (code, out, err) = unset_override_with(&host, &dry_run()); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!(out, SHELL_UNSET, "no plan is printed"); + assert!( + err.starts_with("Could not read the current User HSA_OVERRIDE_GFX_VERSION"), + "{err}" + ); + } + + /// fix-2 reads the Machine scope before `--dry-run` is consulted too, so a + /// Machine read that fails stops a dry run with no plan, even with the + /// User value read and set. + #[test] + fn unset_override_dry_run_with_a_failed_machine_read_stops_without_a_plan() { + let host = FakeHost::new((exited(0), "10.3.0"), exited(0)) + .with_machine_read((CommandOutcome::NotFound, "")); + let (code, out, err) = unset_override_with(&host, &dry_run()); + assert_eq!(code, exit::FAILED); + assert!(host.writes().is_empty()); + assert_eq!(out, SHELL_UNSET, "no plan is printed"); + assert_eq!( + err, + "Could not read the current Machine HSA_OVERRIDE_GFX_VERSION: PowerShell could not \ + be started: it was not found; nothing was changed.\n" + ); + } + + /// Every "re-run `rocm fix `" line names a recipe that exists and has + /// a runner, so the recovery it advises can actually be carried out. + #[test] + fn every_rerun_id_names_a_recipe_with_a_runner() { + for id in [RENDER_GROUP_ID, UNSET_OVERRIDE_ID, PATH_ID, IGPU_DGPU_ID] { + let recipe = super::super::catalog::find_recipe(id) + .unwrap_or_else(|| panic!("{id} is not in the catalog")); + assert!(recipe.runner.is_some(), "{id} has no runner to re-run"); + } + } + /// The Windows arm of fix-2 does mutate (it runs `setx` to clear the User /// scope), so unlike the Linux arm above, a decline has to come back as /// `5`, not `0` -- `skills/rocm-doctor/reference.md` documents `5` as @@ -897,41 +1841,75 @@ mod tests { /// this way. #[test] fn windows_decline_on_user_scope_returns_5() { - let code = - report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "", |_| false); + let host = FakeHost::new(unset(), exited(0)); + let code = report_and_clear_override_windows( + &FixOptions::default(), + "10.3.0", + "", + |_| false, + host.runner(), + &mut Vec::new(), + &mut Vec::new(), + ); assert_eq!(code, 5, "a declined User-scope clear must exit 5, not 0"); + assert!(host.writes().is_empty(), "a decline must not run setx"); } /// Declining the User scope must not swallow the Machine-scope guidance: /// a user with both scopes set still needs the elevated-shell /// instructions printed, so the decline can't `return 5` on the spot -- /// it has to fall through and let the Machine-scope block run first. - /// This test can't see stdout, but it pins the return code so a future - /// change that reintroduces an early `return 5` (skipping that block) - /// would have to change this assertion to keep passing. + /// Asserted on the captured stdout together with the exit code, so an + /// early `return 5` that skips the block fails here. #[test] fn windows_decline_with_machine_scope_also_set_still_returns_5() { - let code = - report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "11.0.0", |_| { - false - }); + let host = FakeHost::new(unset(), exited(0)); + let mut out = Vec::new(); + let code = report_and_clear_override_windows( + &FixOptions::default(), + "10.3.0", + "11.0.0", + |_| false, + host.runner(), + &mut out, + &mut Vec::new(), + ); assert_eq!(code, 5); + assert!(host.writes().is_empty(), "a decline must not run setx"); + let out = text(&out); + assert!(out.contains(" Machine scope: 11.0.0\n"), "{out}"); + assert!( + out.ends_with( + "\nThe Machine scope value cannot be cleared without an Admin shell. Either run \ + an elevated PowerShell and execute:\n \ + [Environment]::SetEnvironmentVariable('HSA_OVERRIDE_GFX_VERSION', $null, \ + 'Machine')\nor remove it through System Properties -> Environment Variables -> \ + System variables. This command does NOT elevate itself.\n" + ), + "the elevated-shell guidance must still be printed after a decline:\n{out}" + ); } #[test] fn windows_consent_granted_is_not_treated_as_a_decline() { - // `setx` isn't on this host, so a real run would report failure (`4`) - // -- the point here is only that the consent gate itself was - // reached and answered "yes", not routed as a decline. - let code = - report_and_clear_override_windows(&FixOptions::default(), "10.3.0", "", |_| true); + let host = FakeHost::new(unset(), exited(0)); + let code = report_and_clear_override_windows( + &FixOptions::default(), + "10.3.0", + "", + |_| true, + host.runner(), + &mut Vec::new(), + &mut Vec::new(), + ); - assert_ne!( - code, 5, - "granted consent must never be reported as declined" + assert_eq!( + code, 0, + "granted consent and a setx that exits 0 is success" ); + assert_eq!(host.writes(), [vec!["setx", OVERRIDE_VAR, ""]]); } #[test] @@ -941,18 +1919,36 @@ mod tests { ..FixOptions::default() }; - let code = report_and_clear_override_windows(&opts, "10.3.0", "", |_| { - panic!("dry-run must not reach the consent gate") - }); + let host = FakeHost::new(unset(), exited(0)); + let code = report_and_clear_override_windows( + &opts, + "10.3.0", + "", + |_| panic!("dry-run must not reach the consent gate"), + host.runner(), + &mut Vec::new(), + &mut Vec::new(), + ); + assert!(host.calls().is_empty(), "a dry-run runs nothing"); assert_eq!(code, 0, "a dry-run preview is not a decline"); } #[test] fn windows_nothing_persisted_in_either_scope_returns_0_without_prompting() { - let code = report_and_clear_override_windows(&FixOptions::default(), "", "", |_| { - panic!("nothing to clear means no consent gate at all") - }); + let host = FakeHost::new(unset(), exited(0)); + let mut out = Vec::new(); + let code = report_and_clear_override_windows( + &FixOptions::default(), + "", + "", + |_| panic!("nothing to clear means no consent gate at all"), + host.runner(), + &mut out, + &mut Vec::new(), + ); + assert!(host.calls().is_empty(), "nothing to clear runs nothing"); + assert!(text(&out).contains("You're done"), "{}", text(&out)); assert_eq!(code, 0); } @@ -964,18 +1960,10 @@ mod tests { /// `setx PATH ""` -- replacing the user's entire real PATH with /// one entry. /// - /// `opts.dry_run = true` is the discriminator, not `opts.yes`: on a host - /// without a real `powershell`/`setx` (this Mac included), the actual - /// write fails regardless of whether it was ever reached, so asserting - /// against a "the write failed" code cannot tell "aborted before the - /// write" apart from "attempted the write and the write itself failed" - /// -- both land on the same non-zero code by coincidence of this host's - /// missing binary. Collapsing the failed read to `""` instead falls - /// through to the dry-run short-circuit (which returns `0`, "preview - /// printed, nothing to do"), while the fix aborts at the read itself and - /// never reaches that branch -- so this is the one assertion that is - /// both host-independent and still fails against the unfixed - /// collapse-to-empty behavior. + /// `opts.dry_run = true` is the discriminator: collapsing the failed read + /// to `""` falls through to the dry-run short-circuit (which returns `0`, + /// "preview printed, nothing to do"), while the fix aborts at the read + /// itself and never reaches that branch. #[test] fn windows_path_read_failure_aborts_before_plan_or_write() { let opts = FixOptions { @@ -983,17 +1971,20 @@ mod tests { ..FixOptions::default() }; - let code = path_export_windows(&opts, r"C:\ROCm\bin", || { - None // the read itself failed (PowerShell missing/blocked/timed out) - }); + // The read itself failed (PowerShell missing/blocked/timed out). + let host = FakeHost::new((CommandOutcome::NotFound, ""), exited(0)); + let (code, out, err) = path_export_with(&host, &opts); - assert_ne!( - code, 0, + assert_eq!( + code, + exit::FAILED, "a failed read must not be silently absorbed by the dry-run preview" ); - assert_ne!( - code, 5, - "a failed read was never offered a prompt to decline" + assert!(host.writes().is_empty(), "a failed read writes nothing"); + assert_eq!(out, "", "no plan is printed from a read that failed"); + assert!( + err.starts_with("Could not read the current User PATH: "), + "{err}" ); // The published `failed` description names this case: a read of state // the fix needed, failing even under `--dry-run`. @@ -1102,9 +2093,13 @@ mod tests { #[test] fn windows_path_already_contains_bin_dir_short_circuits_with_no_change() { let bin_dir = r"C:\ROCm\bin"; - let code = path_export_windows(&FixOptions::default(), bin_dir, || { - Some(format!(r"C:\Windows;{bin_dir}")) - }); + let existing = format!(r"C:\Windows;{bin_dir}"); + let host = FakeHost::new((CommandOutcome::Exited(0), &existing), exited(0)); + let (code, out, _) = path_export_with(&host, &FixOptions::default()); + assert_eq!( + out, + format!("User PATH already contains {bin_dir}; no change.\n") + ); assert_eq!( code, 0, @@ -1123,7 +2118,17 @@ mod tests { ..FixOptions::default() }; - let code = path_export_windows(&opts, r"C:\ROCm\bin", || Some(String::new())); + let host = FakeHost::new((exited(0), r"C:\Windows"), exited(0)); + let (code, out, err) = path_export_with(&host, &opts); + assert!(host.writes().is_empty(), "a dry-run writes nothing"); + assert_eq!(host.calls().len(), 1, "only the read runs"); + assert_eq!( + out, + "Plan: prepend C:\\ROCm\\bin to your User PATH:\n \ + [Environment]::SetEnvironmentVariable('PATH', 'C:\\ROCm\\bin;C:\\Windows', 'User')\n\ + (dry-run; not executed)\n" + ); + assert_eq!(err, ""); assert_eq!(code, 0, "a dry-run preview is not a decline"); } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index c2ea229b61..9a722f8f5e 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -83,12 +83,13 @@ use windows_inventory::{ }; #[cfg(target_os = "linux")] use wsl::detect_wsl_windows_display_probe_text; +pub use wsl::ldconfig_cache; #[cfg(test)] use wsl::local_windows_host_driver_probe; pub use wsl::{WSL_ROCDXG_READY, WslHostDriverProbe, is_wsl_host}; pub(crate) use wsl::{ detect_local_windows_host_driver, detect_wsl_host_driver, detect_wsl_summary, is_wsl1_kernel, - ldconfig_cache, ldconfig_lists_librocdxg, + ldconfig_lists_librocdxg, }; use wsl::{detect_wsl_windows_display_gfx_target_fast, wsl_driver_summary}; @@ -139,6 +140,14 @@ pub struct ExamineSummary { pub wsl: Option, pub managed_runtime_count: usize, pub managed_service_count: usize, + /// Managed runtimes' own `librocdxg.so.1` that the runtime's HSA runtime does + /// not load on this WSL host, because it opens `librocdxg.so` and the wheel + /// ships no such file (or the file is a dangling link), so it takes the host's + /// copy instead. Empty off WSL and when every runtime's copy is reachable. See + /// [`crate::runtime_rocdxg`]. Absent from output written before the field + /// existed, hence the default. + #[serde(default)] + pub unlinked_runtime_librocdxg: Vec, pub model_cache_entries: usize, pub config_dir: PathBuf, pub data_dir: PathBuf, @@ -170,7 +179,7 @@ pub struct WslSummary { #[serde(default)] pub librocdxg_on_ld_library_path: bool, /// `librocdxg.so` is in the same `lib` directory as a `libhsa-runtime64`, - /// which finds it through its own `$ORIGIN` runpath with no cache entry and + /// which finds it through its own `$ORIGIN` search path with no cache entry and /// no `LD_LIBRARY_PATH`. Absent from output written before the field /// existed, hence the default. #[serde(default)] @@ -254,6 +263,8 @@ impl ExamineSummary { detect_windows_examine_inventory() }; let wsl = detect_wsl_summary(); + let unlinked_runtime_librocdxg = + crate::runtime_rocdxg::unlinked_managed_runtime_copies(wsl.is_some(), &paths); let detected_gfx_target = detect_examine_gfx_target_fast(windows_inventory.as_ref()); let compatible_therock_family = detected_gfx_target .as_deref() @@ -294,6 +305,7 @@ impl ExamineSummary { &paths.data_dir.join("runtimes").join("registry"), ), managed_service_count: count_json_files(&paths.services_dir()), + unlinked_runtime_librocdxg, model_cache_entries: count_dir_entries(&paths.data_dir.join("models")), config_dir: paths.config_dir, data_dir: paths.data_dir, @@ -592,6 +604,21 @@ mod tests { Ok(()) } + /// `rocm remote doctor` and the daemon read an `ExamineSummary` another build + /// wrote; one from before `unlinked_runtime_librocdxg` existed must still load. + #[test] + fn a_summary_written_before_the_runtime_librocdxg_field_still_loads() { + let mut value = serde_json::to_value(windows_summary_with_driver(None)).expect("serialise"); + let removed = value + .as_object_mut() + .expect("an object") + .remove("unlinked_runtime_librocdxg"); + assert!(removed.is_some(), "the field is serialised today"); + + let loaded: ExamineSummary = serde_json::from_value(value).expect("old summary loads"); + assert!(loaded.unlinked_runtime_librocdxg.is_empty()); + } + pub(super) fn windows_summary_with_driver( info: Option, ) -> ExamineSummary { @@ -620,6 +647,7 @@ mod tests { wsl: None, managed_runtime_count: 0, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index f9fc1f4b66..09d6626eab 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -28,6 +28,38 @@ impl ExamineSummary { } } + /// The `wsl_runtime_librocdxg` line of the examine text, or an empty string. + /// + /// Present only when a managed runtime ships its own `librocdxg.so.1` that its + /// HSA runtime does not load. `driver_status` describes the host's plumbing + /// and stays as it is; this line is about the runtime. Each runtime's fix is + /// the `ln -sfn` command [`crate::runtime_rocdxg::link_command`] builds (`-fn` + /// so it also clears a dangling `librocdxg.so`). It makes the same relative + /// link `rocm install sdk` creates for a runtime it installs (through + /// `link_runtime_copy`, not through the command). + fn runtime_librocdxg_line(&self) -> String { + if self.unlinked_runtime_librocdxg.is_empty() { + return String::new(); + } + let commands = self + .unlinked_runtime_librocdxg + .iter() + .map(|copy| crate::runtime_rocdxg::link_command(copy)) + .collect::>() + .join("; "); + let count = self.unlinked_runtime_librocdxg.len(); + let noun = if count == 1 { "file sits" } else { "files sit" }; + format!( + " wsl_runtime_librocdxg: {count} managed-runtime librocdxg.so.1 {noun} beside libhsa-runtime64 with \ + no librocdxg.so (or a dangling one), and the runtime opens librocdxg.so, so it loads the \ + host's librocdxg if there is one; a host copy older than the runtime expects fails at first GPU \ + use (for example `undefined symbol: hsaKmtSetPersistingCacheSize`). This lists what the files \ + show, not whether this machine is affected. To make the runtime use its own copy (this adds a \ + link inside the runtime's directory, including for a runtime the CLI did not install) run: \ + {commands}\n" + ) + } + pub fn render_text(&self) -> String { let legacy_paths = if self.legacy_rocm.paths.is_empty() { "".to_owned() @@ -50,7 +82,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n{} managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -89,6 +121,7 @@ impl ExamineSummary { wsl.is_some_and(|summary| summary.cargo), wsl.and_then(|summary| summary.detail.as_deref()) .unwrap_or(""), + self.runtime_librocdxg_line(), self.managed_runtime_count, self.managed_service_count, self.model_cache_entries, @@ -163,6 +196,7 @@ mod tests { wsl: None, managed_runtime_count: 2, managed_service_count: 1, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 3, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), @@ -219,6 +253,7 @@ mod tests { wsl: None, managed_runtime_count: 0, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), @@ -293,6 +328,54 @@ mod tests { assert!(!undated.contains(crate::windows_driver::DISPLAY_DRIVER_OLD_FIX_ID)); } + // Pins POSIX path strings; the finding is WSL-only. + #[cfg(unix)] + #[test] + fn examine_names_a_runtime_librocdxg_its_runtime_does_not_load() { + let mut summary = examine_summary_with_gpu(None, None); + summary.unlinked_runtime_librocdxg = vec![PathBuf::from("/rt/lib/librocdxg.so.1")]; + let rendered = summary.render_text(); + // The wording of the line is pinned here; that the command it names clears + // the finding is proved by scenario `examine-51`, which runs it. + assert!( + rendered.contains("wsl_runtime_librocdxg: 1 managed-runtime librocdxg.so.1 file sits beside libhsa-runtime64 with no librocdxg.so (or a dangling one)"), + "{rendered}" + ); + assert!( + rendered.contains("run: ln -sfn librocdxg.so.1 '/rt/lib/librocdxg.so'"), + "{rendered}" + ); + // The host's own plumbing verdict is a separate line and is not rewritten. + assert!(rendered.contains("driver_status: amd_display_driver_detected")); + } + + #[test] + fn examine_prints_no_runtime_librocdxg_line_when_every_runtime_loads_its_own() { + let clean = examine_summary_with_gpu(None, None).render_text(); + assert!(!clean.contains("wsl_runtime_librocdxg"), "{clean}"); + // The same summary with one copy listed does print it, so the absence + // above is the absence of the finding and not of the line. + let mut listed = examine_summary_with_gpu(None, None); + listed.unlinked_runtime_librocdxg = vec![PathBuf::from("/rt/lib/librocdxg.so.1")]; + assert!(listed.render_text().contains("wsl_runtime_librocdxg")); + } + + // Pins POSIX path strings; the finding is WSL-only. + #[cfg(unix)] + #[test] + fn examine_joins_the_commands_for_several_copies() { + let mut summary = examine_summary_with_gpu(None, None); + summary.unlinked_runtime_librocdxg = vec![ + PathBuf::from("/a/lib/librocdxg.so.1"), + PathBuf::from("/b/lib/librocdxg.so.1"), + ]; + let rendered = summary.render_text(); + assert!(rendered.contains("wsl_runtime_librocdxg: 2 managed-runtime librocdxg.so.1 files sit beside libhsa-runtime64 with no librocdxg.so (or a dangling one)")); + assert!(rendered.contains( + "run: ln -sfn librocdxg.so.1 '/a/lib/librocdxg.so'; ln -sfn librocdxg.so.1 '/b/lib/librocdxg.so'" + )); + } + #[test] fn examine_prints_no_driver_age_lines_when_no_version_was_read() { let rendered = windows_summary_with_driver(None).render_text(); @@ -334,6 +417,7 @@ mod tests { wsl: None, managed_runtime_count: 0, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), @@ -388,6 +472,7 @@ mod tests { wsl: None, managed_runtime_count: 0, managed_service_count: 0, + unlinked_runtime_librocdxg: Vec::new(), model_cache_entries: 0, config_dir: PathBuf::from("config"), data_dir: PathBuf::from("data"), diff --git a/crates/rocm-core/src/host_gpu/wsl.rs b/crates/rocm-core/src/host_gpu/wsl.rs index c5900a039a..ac94af0945 100644 --- a/crates/rocm-core/src/host_gpu/wsl.rs +++ b/crates/rocm-core/src/host_gpu/wsl.rs @@ -193,7 +193,7 @@ pub(crate) fn is_wsl1_kernel(kernel_release: &str) -> bool { /// Deliberately not routed through [`host_path`]: these are programs this /// process executes, not files it reads to describe the machine, and a /// simulated host root holds no working linker to run. -pub(crate) fn ldconfig_cache() -> Option { +pub fn ldconfig_cache() -> Option { for program in ["ldconfig", "/sbin/ldconfig", "/usr/sbin/ldconfig"] { if let Some(text) = capture_optional_command(program, &["-p"]) { return Some(text); @@ -258,7 +258,8 @@ fn librocdxg_on_loader_path(ld_library_path: Option<&str>, exists: impl Fn(&Path /// a `libhsa-runtime64`. /// /// The runtime opens `librocdxg.so` by name, and TheRock builds it with -/// `RUNPATH $ORIGIN`, so the loader looks in the runtime's own directory before it +/// an `$ORIGIN` search path (a `DT_RPATH` in the wheel runtimes' `libhsa-runtime64`, +/// per `readelf -d`), so the loader looks in the runtime's own directory before it /// asks the linker cache or `LD_LIBRARY_PATH`. A tarball under /// `/opt/rocm-` keeps the two side by side and registers neither, which /// is why a working install read as missing while those were the only sources diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 36950d0c36..90385d208a 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -40,6 +40,7 @@ pub mod model_readiness; pub mod net; pub mod openmpi; pub mod os_release; +pub mod owner_only_file; mod package_plan; pub mod paths; pub mod placeholder; @@ -53,6 +54,7 @@ pub mod report_rollup; pub mod report_trend; pub mod rocm_install; pub mod runtime; +pub mod runtime_rocdxg; pub mod service_identity; pub mod service_record; mod sigpipe; @@ -61,6 +63,7 @@ pub mod user_path; #[cfg(any(test, feature = "test-support"))] pub mod foreign_temp_probe; +pub mod tensorizer_override; #[cfg(test)] mod test_env; #[cfg(test)] @@ -120,9 +123,10 @@ pub use download::{ pub use endpoint_key::{endpoint_api_key_has_forbidden_chars, generate_endpoint_api_key}; pub use endpoint_readiness::{ EndpointReadiness, EndpointReadinessOutcome, INFERENCE_PROBE_TIMEOUT, - engine_state_inference_verified, managed_service_endpoint_model_ready, + JSON_STATE_LOCK_FILE_NAME, JsonStateFileRefused, JsonStateRefusal, + engine_state_inference_verified, json_state_lock_path, managed_service_endpoint_model_ready, managed_service_endpoint_readiness, merge_json_state_file, openai_chat_completion_probe, - openai_chat_completion_status, openai_models_endpoint_has_model, + openai_chat_completion_status, openai_models_endpoint_has_model, write_json_state_file, }; pub use examine::{ Examination, FrameworkProbe, WSL_PLATFORM_NOTE, gfx_is_apu_family, probe_wsl_distro_from_host, @@ -137,6 +141,7 @@ pub use gfx_target::{ normalize_therock_family, }; pub use hardware_root::host_path; +pub use host_gpu::ldconfig_cache; pub use host_gpu::{ DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, @@ -149,7 +154,7 @@ pub use host_gpu::{ use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; use host_gpu::{ detect_linux_sysfs_gfx_target, detect_local_windows_host_driver, detect_wsl_host_driver, - detect_wsl_summary, is_wsl1_kernel, ldconfig_cache, ldconfig_lists_librocdxg, + detect_wsl_summary, is_wsl1_kernel, ldconfig_lists_librocdxg, }; pub use jsonl_store::jsonl_lock_path; pub use log_tail::{ @@ -158,9 +163,11 @@ pub use log_tail::{ tail_log_decoded, }; pub use managed_runtime::{ - FrameworkInterpreter, ManagedRuntimeEnvironment, active_managed_framework_interpreter, - active_managed_therock_channel, active_managed_therock_environment, - active_managed_therock_version, detect_managed_therock_family, prepend_runtime_paths, + FrameworkInterpreter, ManagedRuntimeEnvironment, RUNTIME_COMPAT_DIR_NAME, + active_managed_framework_interpreter, active_managed_therock_channel, + active_managed_therock_environment, active_managed_therock_version, amd_smi_link_missing, + detect_managed_therock_family, ensure_amd_smi_compat, prepend_runtime_paths, + remove_stale_numa_shim, runtime_loader_path_entries, runtime_loader_path_entries_read_only, }; use managed_runtime::{collect_sdk_library_paths, managed_therock_sdk_probe_candidates}; pub use model_catalog::{ @@ -242,6 +249,9 @@ pub use service_record::{ engine_stop_decides_confirmation, }; pub use sigpipe::with_sigpipe_ignored; +pub use tensorizer_override::{ + TENSORIZER_PYTHON314_FIX, tensorizer_override_requirement, vllm_tensorizer_override_installed, +}; pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, diff --git a/crates/rocm-core/src/managed_runtime.rs b/crates/rocm-core/src/managed_runtime.rs index 341db1e477..b46c0605bb 100644 --- a/crates/rocm-core/src/managed_runtime.rs +++ b/crates/rocm-core/src/managed_runtime.rs @@ -279,6 +279,30 @@ pub(crate) fn managed_therock_environment_records( .collect() } +/// Every library entry the environment builder produces for each managed runtime +/// in the registry (`managed_therock_environment_from_record`): the recorded +/// library paths, each root's `lib`, `lib64` and `lib/rocm_sysdeps/lib` (and, as +/// the builder collects them, its `bin`), over the SDK root, runtime roots and +/// install root, plus, on Linux, the host's `/usr/lib/wsl/lib` when it exists +/// (that one is not a runtime's own). A directory shared by two runtimes appears +/// twice. +/// +/// `rocm install sdk` links the probe's `library_paths`, which the registry record +/// keeps and this list includes (pinned by +/// `runtime_rocdxg::tests::examine_scans_every_directory_the_install_would_link`), +/// so examine scans at least every directory the install links; the two lists are +/// computed separately. +/// +/// For checks that look at what a runtime ships on disk, such as +/// [`crate::runtime_rocdxg`], rather than at the one runtime that is active. +pub(crate) fn managed_runtime_library_dirs(paths: &AppPaths) -> Vec { + let registry_dir = paths.data_dir.join("runtimes").join("registry"); + managed_therock_environment_records(®istry_dir) + .iter() + .flat_map(|(_, record)| managed_therock_environment_from_record(record).library_entries) + .collect() +} + fn managed_therock_environment_from_record( record: &TheRockFamilyManifest, ) -> ManagedRuntimeEnvironment { @@ -481,6 +505,252 @@ pub(crate) fn collect_sdk_library_paths(root: &Path, paths: &mut Vec) { } } +/// The directory, inside a managed Python environment, where rocm-cli keeps the +/// library shims a runtime's processes need on their loader path. +/// +/// It holds the amd-smi link, the `libmpi_cxx` stub and, from older releases, a +/// `libnuma.so.1` link. A caller that puts the directory on a loader path exposes +/// all of them, so a shim here can hide a host library: a new one has to say what +/// it would hide and who clears it (see [`ensure_amd_smi_compat`]). +pub const RUNTIME_COMPAT_DIR_NAME: &str = "rocm-cli-lib-compat"; + +/// The shim directory of the environment `python` belongs to, or `None` for an +/// interpreter that is not in a managed environment's `bin` folder. +fn runtime_compat_dir_for_python(python: &Path) -> Option { + crate::runtime_python_env_root_for_executable(python) + .map(|root| root.join(RUNTIME_COMPAT_DIR_NAME)) +} + +/// Make `libamd_smi.so` resolvable to the runtime's own amd-smi library. +/// +/// The `amdsmi` Python package that ships with the SDK opens `libamd_smi.so` by +/// that bare name, after a lookup relative to its own path that only fits the +/// SDK's own tree and misses in a virtual environment. The SDK ships the library +/// as `libamd_smi.so.` and not under the bare name, so the bare name resolves +/// through the host's loader cache to whatever amd-smi the host has installed (a +/// system ROCm), or else to the copy inside the `amdsmi` package. Either way a +/// second amd-smi library is loaded next to the SDK's own, which torch has already +/// loaded through librccl, and two copies in one process enumerate nothing: +/// `torch.cuda.device_count()` on ROCm counts devices through amd-smi, so it +/// reports 0 while HIP itself sees every device. Measured on an MI300X with SDK +/// 7.14.1 beside a system ROCm 7.2.4. +/// +/// This links the bare name, inside `compat_dir`, to the SDK's own library, so a +/// loader path that carries `compat_dir` makes the package open the library that +/// is already loaded rather than a second one. Returns `compat_dir` when it holds +/// the link, for the caller to add to the loader path. Returns `None`, and writes +/// nothing, when no directory in `library_dirs` holds the library, when any +/// directory in `library_dirs` already holds the bare name (the loader finds it +/// ahead of the link, which goes last), or when the link cannot be made. Where +/// more than one soname is present in a directory, the highest by number is used. +/// +/// The directory is also the one the vLLM launch keeps a `libmpi_cxx.so.40` stub +/// in, and a loader path that carries it would load a leftover stub ahead of a +/// real host `libmpi_cxx`. So before it returns the directory this removes such a +/// stub where the host has a real library (see +/// [`crate::openmpi::remove_stale_mpi_cxx_stub`]), which keeps every caller that +/// adds the directory to its loader path clear of it. It likewise removes a +/// `libnuma.so.1` link an older release wrote there ([`remove_stale_numa_shim`]). +pub fn ensure_amd_smi_compat(compat_dir: &Path, library_dirs: &[PathBuf]) -> Option { + ensure_amd_smi_compat_with( + compat_dir, + library_dirs, + crate::openmpi::real_libmpi_cxx_present, + ) +} + +/// [`ensure_amd_smi_compat`] with the "does the host have a real `libmpi_cxx`" +/// answer injected, for the stub cleanup it gates. +fn ensure_amd_smi_compat_with( + compat_dir: &Path, + library_dirs: &[PathBuf], + real_libmpi_cxx_present: impl FnOnce() -> bool, +) -> Option { + let library = amd_smi_library_needing_bare_name(library_dirs)?; + let dir = crate::openmpi::ensure_compat_symlink(compat_dir, "libamd_smi.so", &library)?; + crate::openmpi::remove_stale_mpi_cxx_stub(&dir, real_libmpi_cxx_present()); + remove_stale_numa_shim(&dir); + Some(dir) +} + +/// Whether `compat_dir` holds the `libnuma.so.1` symlink an older rocm-cli wrote. +/// +/// It pointed at the SDK's bundled `librocm_sysdeps_numa.so.1`, whose renamed +/// symbol versions cannot satisfy the `libnuma_1.2` binding PyTorch's `libc10.so` +/// needs, so on a loader path it hides a correctly installed system libnuma and +/// breaks `import torch`. Only a symlink counts: anything else was not written by +/// rocm-cli. +fn stale_numa_shim_present(compat_dir: &Path) -> bool { + compat_dir + .join("libnuma.so.1") + .symlink_metadata() + .is_ok_and(|meta| meta.file_type().is_symlink()) +} + +/// Remove the link [`stale_numa_shim_present`] describes. +/// +/// For a caller that is about to put `compat_dir` on a loader path. A no-op where +/// there is none. A failed removal is ignored, as for the `libmpi_cxx` stub: the +/// launch carries on, and the diagnostic keeps reporting the directory as not ready. +pub fn remove_stale_numa_shim(compat_dir: &Path) { + if stale_numa_shim_present(compat_dir) { + let _ = fs::remove_file(compat_dir.join("libnuma.so.1")); + } +} + +/// The loader-path directories for a managed runtime's `python`: its recorded +/// `library_paths`, plus the shim directory when [`ensure_amd_smi_compat`] made +/// one. +/// +/// What a caller that spawns the runtime's interpreter, outside an engine launch, +/// puts on `LD_LIBRARY_PATH`. Exists so each such caller gets the same set. +pub fn runtime_loader_path_entries(python: &Path, library_paths: &[PathBuf]) -> Vec { + runtime_loader_path_entries_with( + python, + library_paths, + crate::openmpi::real_libmpi_cxx_present, + ) +} + +fn runtime_loader_path_entries_with( + python: &Path, + library_paths: &[PathBuf], + real_libmpi_cxx_present: impl FnOnce() -> bool, +) -> Vec { + let mut entries = library_paths.to_vec(); + if let Some(shim) = runtime_compat_dir_for_python(python) + .and_then(|dir| ensure_amd_smi_compat_with(&dir, library_paths, real_libmpi_cxx_present)) + { + entries.push(shim); + } + entries +} + +/// [`runtime_loader_path_entries`] without the write. +/// +/// The shim directory is added only where an install or a launch already left it as +/// they would: the link pointing at the library [`ensure_amd_smi_compat`] would +/// pick, no leftover `libmpi_cxx` stub beside it that a real host library should +/// win over, and no `libnuma.so.1` link an older release left. For a diagnostic, +/// which must not change the environment it reports on, and so must not report a +/// link as healthy that a launch would rewrite. +pub fn runtime_loader_path_entries_read_only( + python: &Path, + library_paths: &[PathBuf], +) -> Vec { + runtime_loader_path_entries_read_only_with( + python, + library_paths, + crate::openmpi::real_libmpi_cxx_present, + ) +} + +fn runtime_loader_path_entries_read_only_with( + python: &Path, + library_paths: &[PathBuf], + real_libmpi_cxx_present: impl FnOnce() -> bool, +) -> Vec { + let mut entries = library_paths.to_vec(); + entries.extend(amd_smi_ready_dir( + python, + library_paths, + real_libmpi_cxx_present, + )); + entries +} + +/// Whether `python`'s runtime needs the amd-smi link and has no usable one. +/// +/// The SDK ships `libamd_smi.so.` without the bare name ([`ensure_amd_smi_compat`]), +/// and the environment's shim directory does not yet hold what a launch would leave +/// (see [`runtime_loader_path_entries_read_only`]). Reads only. For a diagnostic, +/// since a launch that could not make the link says nothing itself. +/// +/// `false` when the interpreter is not in a `bin` or `Scripts` folder: there is +/// then no shim directory for this diagnostic, the ComfyUI launch or the install +/// probe to name, so none of them could write the link. The vLLM launch derives +/// its own directory and can write one where this does not look. +pub fn amd_smi_link_missing(python: &Path, library_paths: &[PathBuf]) -> bool { + let Some(library) = amd_smi_library_needing_bare_name(library_paths) else { + return false; + }; + runtime_compat_dir_for_python(python).is_some_and(|dir| { + !amd_smi_link_dir_ready(&dir, &library, crate::openmpi::real_libmpi_cxx_present) + }) +} + +/// The shim directory of `python`'s environment when it already holds what +/// [`ensure_amd_smi_compat`] would leave there and is safe on a loader path, else +/// `None`. Reads only. +fn amd_smi_ready_dir( + python: &Path, + library_dirs: &[PathBuf], + real_libmpi_cxx_present: impl FnOnce() -> bool, +) -> Option { + let library = amd_smi_library_needing_bare_name(library_dirs)?; + let dir = runtime_compat_dir_for_python(python)?; + amd_smi_link_dir_ready(&dir, &library, real_libmpi_cxx_present).then_some(dir) +} + +/// Whether `dir` holds the link to `library` and nothing that would hide a host +/// library: no stub a real host `libmpi_cxx` should win over, no stale `libnuma` +/// link. +fn amd_smi_link_dir_ready( + dir: &Path, + library: &Path, + real_libmpi_cxx_present: impl FnOnce() -> bool, +) -> bool { + // The write path repoints a link that goes anywhere else, so one that does is + // not healthy. + fs::read_link(dir.join("libamd_smi.so")).is_ok_and(|target| target == library) + && !crate::openmpi::mpi_cxx_stub_shadows_real_library(dir, real_libmpi_cxx_present()) + && !stale_numa_shim_present(dir) +} + +/// The library the bare name has to point at, or `None` when some directory +/// already holds the bare name or none holds a `libamd_smi.so.`. +/// +/// Directories are taken in order and the first one holding a soname decides, the +/// highest-numbered one inside it. That assumes the SDK records a single amd-smi +/// soname across its directories; where two differ, the first directory's wins +/// rather than the highest number. +fn amd_smi_library_needing_bare_name(library_dirs: &[PathBuf]) -> Option { + // A bare name in any directory is found by the loader before the shim, which + // sits last on the path, so no shim is needed whichever directory holds it. + if library_dirs + .iter() + .any(|dir| dir.join("libamd_smi.so").exists()) + { + return None; + } + for dir in library_dirs { + let Ok(entries) = fs::read_dir(dir) else { + continue; + }; + // The soname (`libamd_smi.so.26`): digits only after `.so.`, so a fully + // versioned file (`.so.26.2.2`) beside it is not taken for it. Where an + // in-place upgrade left an older soname beside the current one, take the + // highest by number, not by string (`.so.8` sorts after `.so.26`). + let soname = entries + .flatten() + .filter_map(|entry| { + let version: u64 = entry + .file_name() + .to_str()? + .strip_prefix("libamd_smi.so.")? + .parse() + .ok()?; + Some((version, entry.path())) + }) + .max_by_key(|(version, _)| *version) + .map(|(_, path)| path); + if soname.is_some() { + return soname; + } + } + None +} + #[derive(Debug, Clone)] pub(crate) struct TheRockSdkProbeCandidate { pub(crate) root_path: PathBuf, @@ -1189,4 +1459,387 @@ mod tests { fs::remove_dir_all(root).ok(); Ok(()) } + + /// The SDK ships `libamd_smi.so.` only; the bare name has to be made. + #[cfg(target_os = "linux")] + #[test] + fn amd_smi_shim_links_the_bare_name_to_the_sdks_own_library() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-links"); + let sdk_lib = root.join("sdk").join("lib"); + let compat = root.join("env").join(RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.1"), b"older")?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + // The fully versioned file and the static archive beside it are not the + // soname a dependent asks for. + fs::write(sdk_lib.join("libamd_smi.so.26.2.2"), b"sdk")?; + fs::write(sdk_lib.join("libamd_smi_static.a"), b"sdk")?; + + let shim = ensure_amd_smi_compat(&compat, std::slice::from_ref(&sdk_lib)); + + assert_eq!(shim.as_deref(), Some(compat.as_path())); + assert_eq!( + fs::read_link(compat.join("libamd_smi.so"))?, + sdk_lib.join("libamd_smi.so.26"), + "the bare name must open the library torch has already loaded, not a second copy" + ); + // Run again over a link that is already right, and over one left dangling + // by an earlier install of another version. + assert_eq!( + ensure_amd_smi_compat(&compat, std::slice::from_ref(&sdk_lib)).as_deref(), + Some(compat.as_path()) + ); + fs::remove_file(compat.join("libamd_smi.so"))?; + std::os::unix::fs::symlink( + sdk_lib.join("libamd_smi.so.25"), + compat.join("libamd_smi.so"), + )?; + assert_eq!( + ensure_amd_smi_compat(&compat, std::slice::from_ref(&sdk_lib)).as_deref(), + Some(compat.as_path()) + ); + assert_eq!( + fs::read_link(compat.join("libamd_smi.so"))?, + sdk_lib.join("libamd_smi.so.26") + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// Versions compare by number: `.so.8` is older than `.so.26` though it sorts + /// after it, and a bare name in a later directory is found by the loader + /// before the shim, so it needs none. + #[cfg(target_os = "linux")] + #[test] + fn amd_smi_shim_picks_the_highest_soname_and_yields_to_a_later_bare_name() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-numeric"); + let first = root.join("first"); + let later = root.join("later"); + let compat = root.join("env").join(RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&first)?; + fs::create_dir_all(&later)?; + fs::write(first.join("libamd_smi.so.8"), b"stale")?; + fs::write(first.join("libamd_smi.so.26"), b"sdk")?; + + let dirs = [first.clone(), later.clone()]; + assert!(ensure_amd_smi_compat(&compat, &dirs).is_some()); + assert_eq!( + fs::read_link(compat.join("libamd_smi.so"))?, + first.join("libamd_smi.so.26") + ); + + fs::write(later.join("libamd_smi.so"), b"bare")?; + let other = root.join("env2").join(RUNTIME_COMPAT_DIR_NAME); + assert_eq!(ensure_amd_smi_compat(&other, &dirs), None); + assert!(!other.join("libamd_smi.so").exists()); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// Pins the current policy, not a layout the SDK is known to ship: the + /// directory order decides first, so an earlier directory's soname wins over a + /// numerically higher one in a later directory, and the highest number counts + /// only among the sonames inside the directory that wins. + #[cfg(target_os = "linux")] + #[test] + fn amd_smi_soname_comes_from_the_first_directory_that_has_one() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-dir-order"); + let first = root.join("first"); + let later = root.join("later"); + fs::create_dir_all(&first)?; + fs::create_dir_all(&later)?; + fs::write(first.join("libamd_smi.so.8"), b"lower, earlier directory")?; + fs::write( + first.join("libamd_smi.so.9"), + b"highest in the earlier directory", + )?; + fs::write( + later.join("libamd_smi.so.26"), + b"higher number, later directory", + )?; + + assert_eq!( + amd_smi_library_needing_bare_name(&[first.clone(), later.clone()]), + Some(first.join("libamd_smi.so.9")) + ); + // Without the earlier directory the later one is used. + assert_eq!( + amd_smi_library_needing_bare_name(std::slice::from_ref(&later)), + Some(later.join("libamd_smi.so.26")) + ); + // An earlier directory with no soname, or one that cannot be read, is + // skipped rather than deciding. + let empty = root.join("empty"); + fs::create_dir_all(&empty)?; + assert_eq!( + amd_smi_library_needing_bare_name(&[empty, root.join("missing"), later.clone()]), + Some(later.join("libamd_smi.so.26")) + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A `libnuma.so.1` link an older release left here would hide the host's + /// libnuma on every loader path that includes the directory. A launch clears + /// it, and the diagnostic holds the directory back and writes nothing. A file + /// that is not a symlink was not written by rocm-cli and stays. + #[cfg(target_os = "linux")] + #[test] + fn a_stale_numa_link_is_cleared_by_a_launch_and_held_back_by_a_diagnostic() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-stale-numa"); + let env_root = root.join("env"); + let python = env_root.join("bin").join("python"); + let compat = env_root.join(RUNTIME_COMPAT_DIR_NAME); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::create_dir_all(&compat)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + let libs = std::slice::from_ref(&sdk_lib); + let numa = compat.join("libnuma.so.1"); + + assert!(ensure_amd_smi_compat_with(&compat, libs, || false).is_some()); + std::os::unix::fs::symlink("/does/not/exist", &numa)?; + + // The diagnostic: the link is right, but the numa link shadows a host + // library, so the directory is held back and left as it was. + assert_eq!( + runtime_loader_path_entries_read_only_with(&python, libs, || false), + vec![sdk_lib.clone()] + ); + assert!(amd_smi_link_missing(&python, libs)); + assert!(numa.symlink_metadata().is_ok()); + + // A launch clears it, and the diagnostic then accepts the directory. + assert!(ensure_amd_smi_compat_with(&compat, libs, || false).is_some()); + assert!(numa.symlink_metadata().is_err()); + assert!(!amd_smi_link_missing(&python, libs)); + + // Not a symlink: not ours, left alone, and no reason to hold back. + fs::write(&numa, b"a real file")?; + assert!(ensure_amd_smi_compat_with(&compat, libs, || false).is_some()); + assert!(numa.exists()); + assert!(!amd_smi_link_missing(&python, libs)); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// An interpreter outside a managed environment's `bin` folder has no shim + /// directory, so the diagnostic does not call the link missing for it, where + /// the same library set is reported missing for a managed one. + #[cfg(target_os = "linux")] + #[test] + fn amd_smi_link_is_not_reported_missing_for_an_unmanaged_interpreter() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-link-unmanaged"); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + let libs = std::slice::from_ref(&sdk_lib); + + let managed = root.join("env").join("bin").join("python"); + let unmanaged = root.join("python"); + assert!(amd_smi_link_missing(&managed, libs)); + assert!(!amd_smi_link_missing(&unmanaged, libs)); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A library directory that already holds the bare name needs no shim: the + /// loader finds it there before it consults the host's cache. + #[cfg(target_os = "linux")] + #[test] + fn amd_smi_shim_is_not_written_when_the_sdk_ships_the_bare_name() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-bare"); + let sdk_lib = root.join("sdk").join("lib"); + let compat = root.join("env").join(RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + std::os::unix::fs::symlink("libamd_smi.so.26", sdk_lib.join("libamd_smi.so"))?; + + assert_eq!(ensure_amd_smi_compat(&compat, &[sdk_lib]), None); + assert!( + !compat.exists(), + "nothing may be written when no shim is due" + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// No amd-smi library in any recorded directory, or none recorded at all. + #[test] + fn amd_smi_shim_is_not_written_without_an_sdk_library() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-none"); + let empty = root.join("sdk").join("lib"); + let compat = root.join("env").join(RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&empty)?; + fs::write(empty.join("libamd_smi_static.a"), b"x")?; + + assert_eq!( + ensure_amd_smi_compat(&compat, &[empty, root.join("missing")]), + None + ); + assert_eq!(ensure_amd_smi_compat(&compat, &[]), None); + assert!(!compat.exists()); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// What an interpreter outside an engine launch gets on its loader path: the + /// recorded directories first, the shim directory after them. + #[cfg(target_os = "linux")] + #[test] + fn runtime_loader_path_entries_append_the_shim_directory() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-entries"); + let env_root = root.join("env"); + let python = env_root.join("bin").join("python"); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + + let entries = runtime_loader_path_entries(&python, std::slice::from_ref(&sdk_lib)); + assert_eq!( + entries, + vec![sdk_lib.clone(), env_root.join(RUNTIME_COMPAT_DIR_NAME)] + ); + // An interpreter outside a managed environment has nowhere to keep a shim. + assert_eq!( + runtime_loader_path_entries(&root.join("python"), std::slice::from_ref(&sdk_lib)), + vec![sdk_lib] + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// Only a fully versioned file is not the soname a dependent asks for. + #[test] + fn amd_smi_shim_is_not_written_for_a_fully_versioned_file_alone() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-versioned-only"); + let sdk_lib = root.join("sdk").join("lib"); + let compat = root.join("env").join(RUNTIME_COMPAT_DIR_NAME); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26.2.2"), b"sdk")?; + assert_eq!(ensure_amd_smi_compat(&compat, &[sdk_lib]), None); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A diagnostic reads the shim directory where it exists and never makes it. + #[cfg(target_os = "linux")] + #[test] + fn the_read_only_entries_never_write_the_shim() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-read-only"); + let env_root = root.join("env"); + let python = env_root.join("bin").join("python"); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + + let before = runtime_loader_path_entries_read_only(&python, std::slice::from_ref(&sdk_lib)); + assert_eq!(before, vec![sdk_lib.clone()]); + assert!(!env_root.exists(), "a diagnostic must not create anything"); + + runtime_loader_path_entries(&python, std::slice::from_ref(&sdk_lib)); + let after = runtime_loader_path_entries_read_only(&python, std::slice::from_ref(&sdk_lib)); + assert_eq!(after, vec![sdk_lib, env_root.join(RUNTIME_COMPAT_DIR_NAME)]); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A `libmpi_cxx.so.40` stub in the shim directory shadows a real host + /// `libmpi_cxx` once the directory is on a loader path, and the amd-smi link + /// puts it there for every caller. So the shared helpers clear the stub where + /// the host has a real library, and leave it where the host has none (it may + /// be the library torch is loading then). + #[cfg(target_os = "linux")] + #[test] + fn the_shared_helpers_clear_a_leftover_mpi_cxx_stub_only_beside_a_real_library() -> Result<()> { + for (real_libmpi_cxx, stub_remains) in [(true, false), (false, true)] { + let (root, _) = temp_app_paths("amd-smi-shim-stub"); + let env_root = root.join("env"); + let python = env_root.join("bin").join("python"); + let compat = env_root.join(RUNTIME_COMPAT_DIR_NAME); + let stub = compat.join("libmpi_cxx.so.40"); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + let libs = std::slice::from_ref(&sdk_lib); + + // The helper a launch calls with the directory. + fs::create_dir_all(&compat)?; + fs::write(&stub, b"stub")?; + assert_eq!( + ensure_amd_smi_compat_with(&compat, libs, || real_libmpi_cxx).as_deref(), + Some(compat.as_path()) + ); + assert_eq!( + stub.exists(), + stub_remains, + "real host library: {real_libmpi_cxx}" + ); + + // The helper the install device probe calls with an interpreter. + fs::write(&stub, b"stub")?; + let entries = runtime_loader_path_entries_with(&python, libs, || real_libmpi_cxx); + assert_eq!(entries, vec![sdk_lib.clone(), compat.clone()]); + assert_eq!( + stub.exists(), + stub_remains, + "real host library: {real_libmpi_cxx}" + ); + fs::remove_dir_all(root).ok(); + } + Ok(()) + } + + /// A diagnostic holds the shim directory back where a launch would not leave + /// it as it is: a link that goes to another library (an in-place SDK upgrade), + /// or a stub that a real host library should win over. It writes nothing + /// either way. + #[cfg(target_os = "linux")] + #[test] + fn the_read_only_entries_do_not_trust_a_link_or_a_stub_a_launch_would_change() -> Result<()> { + let (root, _) = temp_app_paths("amd-smi-shim-read-only-stale"); + let env_root = root.join("env"); + let python = env_root.join("bin").join("python"); + let compat = env_root.join(RUNTIME_COMPAT_DIR_NAME); + let sdk_lib = root.join("sdk").join("lib"); + fs::create_dir_all(&sdk_lib)?; + fs::create_dir_all(&compat)?; + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk")?; + fs::write(root.join("older.so.25"), b"older")?; + let libs = std::slice::from_ref(&sdk_lib); + let read = |real: bool| runtime_loader_path_entries_read_only_with(&python, libs, || real); + let link = compat.join("libamd_smi.so"); + + // Another SDK's library, and one that is gone. + for target in [root.join("older.so.25"), root.join("gone")] { + std::os::unix::fs::symlink(&target, &link)?; + assert_eq!( + read(false), + vec![sdk_lib.clone()], + "link to {}", + target.display() + ); + assert!(amd_smi_link_missing(&python, libs)); + fs::remove_file(&link)?; + } + + // The right link, and then a stub beside it. + std::os::unix::fs::symlink(sdk_lib.join("libamd_smi.so.26"), &link)?; + let healthy = vec![sdk_lib.clone(), compat.clone()]; + assert_eq!(read(true), healthy); + assert_eq!(read(false), healthy); + fs::write(compat.join("libmpi_cxx.so.40"), b"stub")?; + assert_eq!( + read(false), + healthy, + "no real host library: the stub may be in use" + ); + assert_eq!(read(true), vec![sdk_lib.clone()]); + assert!( + compat.join("libmpi_cxx.so.40").exists(), + "a diagnostic never removes it" + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } } diff --git a/crates/rocm-core/src/openmpi.rs b/crates/rocm-core/src/openmpi.rs index 2f45df2da2..b649016d91 100644 --- a/crates/rocm-core/src/openmpi.rs +++ b/crates/rocm-core/src/openmpi.rs @@ -35,9 +35,7 @@ //! `build_libatomic_install_plan`, `build_libnuma_install_plan`) and the hints //! that name each runtime. -#[cfg(target_os = "linux")] -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::Command; @@ -247,6 +245,10 @@ fn symlink_points_to(link: &Path, target: &Path) -> bool { std::fs::read_link(link).is_ok_and(|dest| dest == target) } +/// File name of the `libmpi_cxx` stub [`ensure_mpi_cxx_compat`] writes into a compat +/// directory. +pub const MPI_CXX_STUB_NAME: &str = "libmpi_cxx.so.40"; + /// Prebuilt `libmpi_cxx.so.40` stub defining the legacy OpenMPI C++ binding /// symbols PyTorch references (see `build.rs`). Empty when the build host had no /// C compiler, in which case the shim is disabled. @@ -313,7 +315,48 @@ pub fn ensure_mpi_cxx_compat(compat_dir: &Path) -> Option { if MPI_CXX_STUB.is_empty() { return None; } - write_compat_file(compat_dir, "libmpi_cxx.so.40", MPI_CXX_STUB) + write_compat_file(compat_dir, MPI_CXX_STUB_NAME, MPI_CXX_STUB) +} + +/// Whether the host has a real `libmpi_cxx.so*`. +/// +/// That makes the stub [`ensure_mpi_cxx_compat`] writes unnecessary. It tells that +/// case apart from the other reasons that function answers `None` (no stub +/// embedded, stub not writable), where an existing stub may still be in use. +#[cfg(target_os = "linux")] +pub fn real_libmpi_cxx_present() -> bool { + find_libmpi_cxx().is_some() +} + +/// Whether `compat_dir` holds a `libmpi_cxx.so.40` stub that shadows a real library. +/// +/// That is a stub an earlier run wrote, on a host that has since gained a real +/// `libmpi_cxx`. The caller passes whether the host has one, from +/// [`real_libmpi_cxx_present`]. For a caller that puts `compat_dir` on a loader +/// path and may not write. +pub fn mpi_cxx_stub_shadows_real_library(compat_dir: &Path, real_libmpi_cxx_present: bool) -> bool { + real_libmpi_cxx_present + && compat_dir + .join(MPI_CXX_STUB_NAME) + .symlink_metadata() + .is_ok() +} + +/// Remove the stub [`mpi_cxx_stub_shadows_real_library`] describes. +/// +/// A no-op when the host has no real `libmpi_cxx`, because then the stub may be the +/// library torch is loading (see [`real_libmpi_cxx_present`]), and when no stub is +/// there. For a caller that is about to put `compat_dir` on a loader path. +pub fn remove_stale_mpi_cxx_stub(compat_dir: &Path, real_libmpi_cxx_present: bool) { + if real_libmpi_cxx_present { + let _ = std::fs::remove_file(compat_dir.join(MPI_CXX_STUB_NAME)); + } +} + +/// Non-Linux hosts have no stub to tell apart. +#[cfg(not(target_os = "linux"))] +pub const fn real_libmpi_cxx_present() -> bool { + false } /// Non-Linux hosts never need the OpenMPI C++ bindings shim. @@ -326,10 +369,11 @@ pub const fn ensure_mpi_cxx_compat(_compat_dir: &std::path::Path) -> Option Option { diff --git a/crates/rocm-core/src/owner_only_file.rs b/crates/rocm-core/src/owner_only_file.rs new file mode 100644 index 0000000000..47a46ccf46 --- /dev/null +++ b/crates/rocm-core/src/owner_only_file.rs @@ -0,0 +1,808 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Creating a file that only the current user can open, for secrets such as a +//! served endpoint's API key. +//! +//! [`open_new_owner_only`] creates the file with its access already narrowed, +//! so the secret is never readable by anyone else, not even between the +//! create and a later permission change: +//! +//! - on Unix the file is created with mode `0600`, and the mode is set again +//! on the open handle before the caller writes anything, because the umask +//! can only remove bits at creation and an unusual one could leave the file +//! unreadable by its own owner; +//! - on Windows it is created with a protected DACL whose single entry grants +//! full access to the user the process runs as (the token user, which is not +//! necessarily the file's owner when the process is elevated). Protected means +//! the file does not inherit entries from its directory, so a data directory +//! that other accounts can read does not make the secret readable by them. +//! No entry is added for administrators or `SYSTEM`. +//! +//! The file is created with `create_new` semantics on both platforms: an +//! existing file or symbolic link at `path`, dangling or not, is an error and is +//! never followed or opened, so a caller cannot be made to write a secret +//! through a file or link somebody else prepared. On Windows that takes +//! `FILE_FLAG_OPEN_REPARSE_POINT`, which `std` also passes for `create_new`, +//! and the path goes through the same long-path conversion `std::fs` applies +//! when it opens a file — every path that is not already verbatim or a short +//! drive-absolute or UNC one, a short relative path included, is made absolute +//! and given the `\\?\` prefix — so a path longer than `MAX_PATH` works here +//! as it does for `std::fs`. To replace an existing secret, use +//! [`crate::atomic_write::write_owner_only_file_atomically`], which stages the +//! bytes in a fresh owner-only file and renames it over the old one. + +use std::fs::File; +use std::io; +use std::path::Path; + +/// Create `path` as a new file that only the current user can open. +/// +/// Returns it open for writing. Fails with [`io::ErrorKind::AlreadyExists`] +/// when anything — a file, a directory or a symbolic link — is already at +/// `path`. +pub fn open_new_owner_only(path: &Path) -> io::Result { + create_new_owner_only(path) +} + +/// The DACL [`open_new_owner_only`] gives a file on Windows, in SDDL. +/// +/// A protected DACL (`P`, so no entry is inherited from the directory) holding +/// one entry that allows full file access (`FA`) to `user_sid`. +/// +/// Built on every platform so the shape it promises is unit-tested everywhere, +/// not only on the Windows job. +pub fn owner_only_sddl(user_sid: &str) -> String { + format!("D:P(A;;FA;;;{user_sid})") +} + +/// Test-only switch: when set, [`open_new_owner_only`] returns the file as the +/// create left it, without setting its mode again on the handle, so a test can +/// see what the create itself asked for. Set only in a re-executed test child. +#[cfg(all(unix, test))] +static SKIP_HANDLE_MODE_RESET: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +#[cfg(unix)] +fn create_new_owner_only(path: &Path) -> io::Result { + use std::fs::{OpenOptions, Permissions}; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + + // Mode 0600 as the file comes into existence, narrowed further by the umask + // but never widened. + let file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path)?; + #[cfg(test)] + if SKIP_HANDLE_MODE_RESET.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(file); + } + // The umask can only have removed bits from 0600, never added any, so + // nobody else can open the file even before this call. Setting the mode on + // the handle makes it exactly 0600 whatever the umask was, so the owner can + // always read the secret back. + if let Err(error) = file.set_permissions(Permissions::from_mode(0o600)) { + drop(file); + let _ = std::fs::remove_file(path); + return Err(error); + } + Ok(file) +} + +#[cfg(windows)] +fn create_new_owner_only(path: &Path) -> io::Result { + let sid = win32::current_user_sid()?; + win32::create_new_with_sddl(path, &owner_only_sddl(&sid)) +} + +/// Test support: `None` when the file at `path` has a protected DACL whose +/// only entry grants the current user full access, as [`open_new_owner_only`] +/// creates it; otherwise the DACL found, in SDDL, for the failure message. +/// +/// Compiled only for this crate's tests and under the `test-support` feature, +/// which the apps enable as a dev-dependency so their own Windows tests can +/// check the files they store. +#[cfg(all(windows, any(test, feature = "test-support")))] +#[doc(hidden)] +pub fn owner_only_dacl_mismatch(path: &Path) -> io::Result> { + use win32::test_support::{canonical_sddl, dacl_sddl, flags_and_entries}; + + let actual = dacl_sddl(path)?; + let expected = canonical_sddl(&owner_only_sddl(&win32::current_user_sid()?))?; + let (flags, entries) = flags_and_entries(&actual); + let (_, expected_entries) = flags_and_entries(&expected); + Ok((!(flags.contains('P') && entries == expected_entries)).then_some(actual)) +} + +// UTF-16 units of the path syntax below; all ASCII. +#[cfg(any(windows, test))] +const SEP: u16 = b'\\' as u16; +#[cfg(any(windows, test))] +const ALT_SEP: u16 = b'/' as u16; +#[cfg(any(windows, test))] +const COLON: u16 = b':' as u16; +/// `\\?\` +#[cfg(any(windows, test))] +const VERBATIM_PREFIX: [u16; 4] = [SEP, SEP, b'?' as u16, SEP]; +/// `\??\` +#[cfg(any(windows, test))] +const NT_PREFIX: [u16; 4] = [SEP, b'?' as u16, b'?' as u16, SEP]; +/// `\\.\` +#[cfg(any(windows, test))] +const DEVICE_PREFIX: [u16; 4] = [SEP, SEP, b'.' as u16, SEP]; +/// `\\?\UNC\` +#[cfg(any(windows, test))] +const UNC_PREFIX: [u16; 8] = [ + SEP, + SEP, + b'?' as u16, + SEP, + b'U' as u16, + b'N' as u16, + b'C' as u16, + SEP, +]; +/// The length below which `std` passes a drive-absolute or UNC path to Windows +/// unchanged (the `CreateDirectoryW` limit, the tightest of the `MAX_PATH` +/// family). +#[cfg(any(windows, test))] +const LEGACY_MAX_PATH: usize = 248; + +/// Whether `std` would pass the UTF-16 path `wide` (no NUL) to Windows as it +/// is: a verbatim path, or a short one that is already drive-absolute or UNC. +#[cfg(any(windows, test))] +fn usable_as_is(wide: &[u16]) -> bool { + if wide.starts_with(&VERBATIM_PREFIX) || wide.starts_with(&NT_PREFIX) { + return true; + } + if wide.len() + 1 >= LEGACY_MAX_PATH { + return false; + } + match wide { + [drive, COLON] | [drive, COLON, SEP | ALT_SEP, ..] => *drive != SEP && *drive != ALT_SEP, + [SEP | ALT_SEP, SEP | ALT_SEP, ..] => true, + _ => false, + } +} + +/// The verbatim, NUL-terminated form of `absolute`, a path `GetFullPathNameW` +/// already made absolute and normalized: `C:\x` becomes `\\?\C:\x`, +/// `\\.\x` becomes `\\?\x`, `\\server\share` becomes +/// `\\?\UNC\server\share`, and a path already verbatim is kept. The same +/// mapping `std::fs` applies before it opens a file, to every path +/// [`usable_as_is`] rejects whatever its length. +#[cfg(any(windows, test))] +fn with_verbatim_prefix(absolute: &[u16]) -> Vec { + let (prefix, rest): (&[u16], &[u16]) = match absolute { + [_, COLON, SEP, ..] => (&VERBATIM_PREFIX, absolute), + _ if absolute.starts_with(&DEVICE_PREFIX) => (&VERBATIM_PREFIX, &absolute[4..]), + _ if absolute.starts_with(&VERBATIM_PREFIX) || absolute.starts_with(&NT_PREFIX) => { + (&[], absolute) + } + [SEP, SEP, ..] => (&UNC_PREFIX, &absolute[2..]), + _ => (&[], absolute), + }; + prefix.iter().chain(rest).copied().chain(Some(0)).collect() +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +pub(crate) mod win32 { + //! The Win32 calls behind the Windows side of + //! [`super::open_new_owner_only`]. Every buffer the system allocates for us + //! is released with `LocalFree`, and the token handle is closed, on every + //! path including the error ones. + + use std::ffi::OsString; + use std::fs::File; + use std::io; + use std::os::windows::ffi::{OsStrExt as _, OsStringExt as _}; + use std::os::windows::io::{FromRawHandle as _, OwnedHandle}; + use std::path::Path; + use std::ptr; + + use windows_sys::Win32::Foundation::{GENERIC_WRITE, HANDLE, INVALID_HANDLE_VALUE, LocalFree}; + use windows_sys::Win32::Security::Authorization::{ + ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, + SDDL_REVISION_1, + }; + use windows_sys::Win32::Security::{ + GetTokenInformation, PSECURITY_DESCRIPTOR, SECURITY_ATTRIBUTES, TOKEN_QUERY, TOKEN_USER, + TokenUser, + }; + use windows_sys::Win32::Storage::FileSystem::{ + CREATE_NEW, CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + }; + use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + + /// A buffer the system allocated with `LocalAlloc` on our behalf, released + /// with `LocalFree` when dropped. + pub(crate) struct LocalBuffer(pub(crate) *mut core::ffi::c_void); + + impl Drop for LocalBuffer { + fn drop(&mut self) { + if !self.0.is_null() { + // SAFETY: the pointer came from an API documented to allocate + // it with `LocalAlloc` and hand ownership to the caller, and + // this guard is its only owner, so it is freed exactly once. + unsafe { + LocalFree(self.0); + } + } + } + } + + /// Read a NUL-terminated UTF-16 string the system allocated. + /// + /// # Safety + /// `wide` must point to a valid NUL-terminated UTF-16 string. + pub(crate) unsafe fn string_from_wide(wide: *const u16) -> io::Result { + let mut len = 0; + // SAFETY: the caller guarantees a NUL terminator, so every offset read + // here up to and including it is inside the string. + while unsafe { *wide.add(len) } != 0 { + len += 1; + } + // SAFETY: `len` units starting at `wide` were just read above. + let units = unsafe { std::slice::from_raw_parts(wide, len) }; + OsString::from_wide(units) + .into_string() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "string is not valid Unicode")) + } + + /// `path` as the NUL-terminated UTF-16 name `std::fs` would hand + /// `CreateFileW`: unchanged when it is already verbatim, or short and + /// drive-absolute or UNC; otherwise made absolute and given the verbatim + /// prefix, which lifts the `MAX_PATH` limit. That includes a short relative + /// path: when it opens a file `std` prefers the verbatim form, unlike + /// [`std::path::absolute`], which returns the plain absolute form. `std` + /// keeps this conversion private, so it is repeated here on top of the + /// public [`std::path::absolute`]; the decisions are in + /// [`super::usable_as_is`] and [`super::with_verbatim_prefix`], which are + /// tested on every host. + pub(crate) fn long_path_wide(path: &Path) -> io::Result> { + let wide: Vec = path.as_os_str().encode_wide().collect(); + if wide.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "strings passed to WinAPI cannot contain NULs", + )); + } + if super::usable_as_is(&wide) { + return Ok(wide.into_iter().chain(Some(0)).collect()); + } + let absolute: Vec = std::path::absolute(path)? + .as_os_str() + .encode_wide() + .collect(); + Ok(super::with_verbatim_prefix(&absolute)) + } + + /// The string form (`S-1-5-21-…`) of the SID of the user this process runs + /// as, read from the process token. + pub(crate) fn current_user_sid() -> io::Result { + let mut raw_token: HANDLE = ptr::null_mut(); + // SAFETY: `GetCurrentProcess` returns a pseudo-handle that needs no + // closing; `raw_token` is a valid out-pointer for the call. + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut raw_token) } == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call succeeded, so `raw_token` is an open handle that we + // own; `OwnedHandle` closes it with `CloseHandle` when dropped. + let token = unsafe { OwnedHandle::from_raw_handle(raw_token) }; + let token_handle: HANDLE = std::os::windows::io::AsRawHandle::as_raw_handle(&token); + + let mut needed = 0u32; + // SAFETY: a null buffer of length 0 asks only for the required size, + // which is written to `needed`. The call reports failure + // (`ERROR_INSUFFICIENT_BUFFER`) by design, so its result is not checked; + // a zero `needed` below is what means it really failed. + unsafe { + GetTokenInformation(token_handle, TokenUser, ptr::null_mut(), 0, &raw mut needed); + } + if needed == 0 { + return Err(io::Error::last_os_error()); + } + // `u64` units so the buffer is aligned for `TOKEN_USER`, which holds a + // pointer. + let mut buffer = vec![0u64; (needed as usize).div_ceil(size_of::())]; + // SAFETY: `buffer` holds at least `needed` writable bytes and outlives + // the call. + if unsafe { + GetTokenInformation( + token_handle, + TokenUser, + buffer.as_mut_ptr().cast(), + needed, + &raw mut needed, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call succeeded, so the buffer starts with a `TOKEN_USER` + // whose SID pointer points into the same buffer, still alive here. + let sid = unsafe { (*buffer.as_ptr().cast::()).User.Sid }; + + let mut wide: *mut u16 = ptr::null_mut(); + // SAFETY: `sid` is a valid SID (see above) and `wide` a valid + // out-pointer. + if unsafe { ConvertSidToStringSidW(sid, &raw mut wide) } == 0 { + return Err(io::Error::last_os_error()); + } + let wide = LocalBuffer(wide.cast()); + // SAFETY: on success the system returned a NUL-terminated string. + unsafe { string_from_wide(wide.0.cast()) } + // `token` and `wide` are released here, on this path and on every + // early return above. + } + + /// A self-relative security descriptor parsed from `sddl`, owned by the + /// returned guard. + pub(crate) fn security_descriptor_from_sddl(sddl: &str) -> io::Result { + let sddl_wide: Vec = sddl.encode_utf16().chain(Some(0)).collect(); + let mut descriptor: PSECURITY_DESCRIPTOR = ptr::null_mut(); + // SAFETY: `sddl_wide` is a NUL-terminated UTF-16 string alive for the + // call; `descriptor` is a valid out-pointer and the size is not wanted. + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl_wide.as_ptr(), + SDDL_REVISION_1, + &raw mut descriptor, + ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(LocalBuffer(descriptor)) + } + + /// Create `path` as a new file whose security descriptor is `sddl`, and + /// return it open for writing. The descriptor is applied by the create + /// itself, so there is no moment at which the file has any other DACL. + pub(crate) fn create_new_with_sddl(path: &Path, sddl: &str) -> io::Result { + let descriptor = security_descriptor_from_sddl(sddl)?; + let attributes = SECURITY_ATTRIBUTES { + nLength: u32::try_from(size_of::()) + .expect("SECURITY_ATTRIBUTES fits in a u32"), + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + let path_wide = long_path_wide(path)?; + // SAFETY: `path_wide` is NUL-terminated and `attributes` points at a + // descriptor that `descriptor` keeps alive past the call. The sharing + // mode, the long-path form of the name and `FILE_FLAG_OPEN_REPARSE_POINT` + // are what `std::fs::OpenOptions` uses for `create_new`, so the handle + // behaves like one `std` would have opened: with that flag `CREATE_NEW` + // fails on a symbolic link at `path`, dangling or not, instead of + // creating the file where the link points. + let handle = unsafe { + CreateFileW( + path_wide.as_ptr(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &raw const attributes, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OPEN_REPARSE_POINT, + ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: `handle` is a freshly opened file handle owned by nobody + // else; `File` takes ownership and closes it. + Ok(unsafe { File::from_raw_handle(handle) }) + } + + #[cfg(any(test, feature = "test-support"))] + pub(crate) mod test_support { + //! Reading a file's DACL back, for the Windows tests here, in + //! `atomic_write`, and (through [`super::super::owner_only_dacl_mismatch`]) + //! in the crates that store secrets with this module. + + use super::{LocalBuffer, long_path_wide, security_descriptor_from_sddl, string_from_wide}; + use std::io; + use std::path::Path; + use std::ptr; + use windows_sys::Win32::Security::Authorization::{ + ConvertSecurityDescriptorToStringSecurityDescriptorW, GetNamedSecurityInfoW, + SDDL_REVISION_1, SE_FILE_OBJECT, + }; + use windows_sys::Win32::Security::{DACL_SECURITY_INFORMATION, PSECURITY_DESCRIPTOR}; + + fn dacl_sddl_of_descriptor(descriptor: PSECURITY_DESCRIPTOR) -> io::Result { + let mut wide: *mut u16 = ptr::null_mut(); + // SAFETY: `descriptor` is a valid security descriptor owned by the + // caller; `wide` is a valid out-pointer and the length is not wanted. + if unsafe { + ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &raw mut wide, + ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + let wide = LocalBuffer(wide.cast()); + // SAFETY: on success the system returned a NUL-terminated string. + unsafe { string_from_wide(wide.0.cast()) } + } + + /// The DACL of the file at `path`, in SDDL (`D:P(A;;FA;;;S-1-…)`). + pub(crate) fn dacl_sddl(path: &Path) -> io::Result { + let path_wide = long_path_wide(path)?; + let mut descriptor: PSECURITY_DESCRIPTOR = ptr::null_mut(); + // SAFETY: `path_wide` is NUL-terminated; the owner, group and SACL + // out-pointers are null because they are not requested, and the + // DACL pointer is not wanted since the descriptor is converted + // whole. + let status = unsafe { + GetNamedSecurityInfoW( + path_wide.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + &raw mut descriptor, + ) + }; + if status != 0 { + return Err(io::Error::from_raw_os_error( + i32::try_from(status).unwrap_or(i32::MAX), + )); + } + let descriptor = LocalBuffer(descriptor); + dacl_sddl_of_descriptor(descriptor.0) + } + + /// `sddl` as Windows itself writes it back, so a comparison is not + /// thrown off by the system abbreviating a well-known SID. + pub(crate) fn canonical_sddl(sddl: &str) -> io::Result { + let descriptor = security_descriptor_from_sddl(sddl)?; + dacl_sddl_of_descriptor(descriptor.0) + } + + /// Split a DACL's SDDL into its control flags (`P`, `AI`, …) and its + /// entries. + pub(crate) fn flags_and_entries(sddl: &str) -> (&str, &str) { + let body = sddl.strip_prefix("D:").unwrap_or(sddl); + body.find('(') + .map_or((body, ""), |start| body.split_at(start)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::unix_time_millis; + use std::io::Write as _; + use std::path::PathBuf; + + fn scratch_dir(label: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "rocm-core-open-new-owner-only-{label}-{}-{}", + std::process::id(), + unix_time_millis() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + root + } + + #[test] + fn owner_only_sddl_is_a_protected_dacl_with_one_full_access_entry() { + assert_eq!( + owner_only_sddl("S-1-5-21-1-2-3-1001"), + "D:P(A;;FA;;;S-1-5-21-1-2-3-1001)" + ); + } + + #[test] + fn an_existing_file_is_never_opened() { + let root = scratch_dir("exists"); + let path = root.join("secret"); + std::fs::write(&path, b"someone else's file").unwrap(); + + let error = open_new_owner_only(&path).expect_err("an existing file must not be opened"); + let contents = std::fs::read(&path).unwrap(); + let _ = std::fs::remove_dir_all(&root); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(contents, b"someone else's file"); + } + + #[cfg(unix)] + #[test] + fn a_symlink_is_never_written_through() { + let root = scratch_dir("symlink"); + let target = root.join("target"); + std::fs::write(&target, b"untouched").unwrap(); + let link = root.join("secret"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let error = open_new_owner_only(&link).expect_err("a link must not be followed"); + let contents = std::fs::read(&target).unwrap(); + let _ = std::fs::remove_dir_all(&root); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(contents, b"untouched"); + } + + /// The file comes out exactly 0600 whatever the umask. The umask is + /// process-wide, so the narrow one this needs is set in a re-executed copy + /// of this test rather than here, where it would reach sibling tests. + /// + /// `0o477` strips the owner's read bit from a plain `mode(0o600)` create, + /// leaving a key file its own owner cannot read back; only the mode set on + /// the handle afterwards brings it to 0600. + #[cfg(unix)] + #[test] + #[allow(unsafe_code)] // libc::umask, in the re-executed child only + fn a_new_file_is_exactly_0600_whatever_the_umask() { + use std::os::unix::fs::PermissionsExt as _; + + if let Some(dir) = std::env::var_os("ROCM_TEST_OWNER_ONLY_UMASK_DIR") { + // SAFETY: `umask` only swaps this process's file-creation mask; + // this child process runs only this one test. + let previous = unsafe { libc::umask(0o477) }; + let path = PathBuf::from(dir).join("secret"); + let opened = open_new_owner_only(&path); + // Put the mask back before anything else creates a file: under + // `cargo llvm-cov` this child writes its own coverage profile at + // exit, and one created under 0477 is unreadable, which crashes + // `llvm-profdata merge` and fails the coverage job. + // SAFETY: as above. + unsafe { + libc::umask(previous); + } + let mut file = opened.unwrap(); + file.write_all(b"key").unwrap(); + drop(file); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "mode under umask 0477 was {mode:o}"); + return; + } + + let root = scratch_dir("umask"); + let status = + std::process::Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "owner_only_file::tests::a_new_file_is_exactly_0600_whatever_the_umask", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_OWNER_ONLY_UMASK_DIR", &root) + .status() + .expect("re-exec the test in a child process"); + let created = root.join("secret").exists(); + let _ = std::fs::remove_dir_all(&root); + assert!(created, "the child did not run the test arm"); + assert!(status.success(), "{status:?}"); + } + + /// Nobody else can open the file even before its mode is set again on the + /// handle: the create itself asks for 0600. Checked in a re-executed child + /// under umask 0, where nothing narrows the requested mode, so what the + /// create asked for is exactly what this sees. + #[cfg(unix)] + #[test] + #[allow(unsafe_code)] // libc::umask, in the re-executed child only + fn the_file_is_owner_only_as_created_before_its_mode_is_set_again() { + use std::os::unix::fs::PermissionsExt as _; + + if let Some(dir) = std::env::var_os("ROCM_TEST_OWNER_ONLY_CREATE_DIR") { + // SAFETY: `umask` only swaps this process's file-creation mask; + // this child process runs only this one test. + let previous = unsafe { libc::umask(0) }; + // Through the real entry point, with only the mode reset on the + // handle switched off, so this sees what the create asked for. + super::SKIP_HANDLE_MODE_RESET.store(true, std::sync::atomic::Ordering::SeqCst); + let path = PathBuf::from(dir).join("secret"); + let created = open_new_owner_only(&path); + // SAFETY: as above; restored before anything else creates a file. + unsafe { + libc::umask(previous); + } + // Switch the reset back on, so nothing else in this process sees + // it off. + super::SKIP_HANDLE_MODE_RESET.store(false, std::sync::atomic::Ordering::SeqCst); + drop(created.unwrap()); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "mode as created under umask 0 was {mode:o}"); + return; + } + + let root = scratch_dir("as-created"); + let status = + std::process::Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "owner_only_file::tests::the_file_is_owner_only_as_created_before_its_mode_is_set_again", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_OWNER_ONLY_CREATE_DIR", &root) + .status() + .expect("re-exec the test in a child process"); + let created = root.join("secret").exists(); + let _ = std::fs::remove_dir_all(&root); + assert!(created, "the child did not run the test arm"); + assert!(status.success(), "{status:?}"); + } + + #[cfg(windows)] + #[test] + fn a_new_file_has_a_protected_dacl_for_the_current_user_only() { + let root = scratch_dir("dacl"); + let path = root.join("secret"); + let mut file = open_new_owner_only(&path).unwrap(); + file.write_all(b"key").unwrap(); + drop(file); + + let mismatch = owner_only_dacl_mismatch(&path).unwrap(); + let contents = std::fs::read(&path).unwrap(); + let _ = std::fs::remove_dir_all(&root); + + assert_eq!(mismatch, None, "the DACL is not protected owner-only"); + assert_eq!(contents, b"key", "the owner cannot read the file back"); + } + + /// The checker itself: a file created with an inherited DACL, as + /// `std::fs::write` leaves it, is reported, so a `None` above means + /// something. + #[cfg(windows)] + #[test] + fn the_dacl_check_reports_an_inherited_dacl() { + let root = scratch_dir("dacl-inherited"); + let path = root.join("plain"); + std::fs::write(&path, b"plain").unwrap(); + + let mismatch = owner_only_dacl_mismatch(&path).unwrap(); + let _ = std::fs::remove_dir_all(&root); + assert!(mismatch.is_some(), "an inherited DACL passed the check"); + } + + /// A symbolic link at the destination — dangling, or pointing at a file — + /// fails the create instead of being followed, so nothing is created or + /// written where it points. Creating a link needs Developer Mode or the + /// symlink privilege; where neither is available the test fails on CI + /// (`CI` set) and elsewhere says so and passes without checking. + #[cfg(windows)] + #[test] + fn a_symlink_is_never_followed_on_windows() { + let root = scratch_dir("symlink-win"); + let missing_target = root.join("missing-target"); + let dangling = root.join("dangling"); + if let Err(error) = std::os::windows::fs::symlink_file(&missing_target, &dangling) { + let _ = std::fs::remove_dir_all(&root); + // On CI this test is the only check of the reparse-point flag, so + // a runner that cannot create links fails rather than passing + // unchecked; a developer machine without the privilege skips. + assert!( + std::env::var_os("CI").is_none(), + "cannot create a symbolic link on CI, so the link-following check did not run: {error}" + ); + eprintln!("SKIPPED: this account cannot create a symbolic link ({error})"); + return; + } + let target = root.join("target"); + std::fs::write(&target, b"untouched").unwrap(); + let live = root.join("live"); + std::os::windows::fs::symlink_file(&target, &live).unwrap(); + + let dangling_result = open_new_owner_only(&dangling).map(drop); + let live_result = open_new_owner_only(&live).map(drop); + let created_at_target = missing_target.exists(); + let target_contents = std::fs::read(&target).unwrap(); + let _ = std::fs::remove_dir_all(&root); + + assert_eq!( + dangling_result.map_err(|error| error.kind()), + Err(io::ErrorKind::AlreadyExists), + "a dangling link was followed" + ); + assert!( + !created_at_target, + "the file was created at the link's target" + ); + assert_eq!( + live_result.map_err(|error| error.kind()), + Err(io::ErrorKind::AlreadyExists), + "a link to a file was followed" + ); + assert_eq!(target_contents, b"untouched"); + } + + /// A destination past `MAX_PATH` is created, as `std::fs` would create it. + #[cfg(windows)] + #[test] + fn a_path_longer_than_max_path_is_created() { + let root = scratch_dir("long-path"); + let mut dir = root.clone(); + while dir.as_os_str().len() < 300 { + dir.push("a-directory-name-that-makes-the-path-long"); + } + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("secret"); + + let created = open_new_owner_only(&path).map(|mut file| file.write_all(b"key")); + let contents = std::fs::read(&path); + let mismatch = owner_only_dacl_mismatch(&path); + let _ = std::fs::remove_dir_all(&root); + + created.unwrap().unwrap(); + assert_eq!(contents.unwrap(), b"key"); + assert_eq!(mismatch.unwrap(), None); + } + + fn wide(text: &str) -> Vec { + text.encode_utf16().collect() + } + + fn wide_nul(text: &str) -> Vec { + text.encode_utf16().chain(Some(0)).collect() + } + + /// The long-path decisions, with expectations written from `std`'s + /// `get_long_path` (its conversion is private, so it cannot be compared + /// at run time): short absolute and UNC paths and verbatim ones pass as + /// they are; anything relative, or long, is converted. + #[test] + fn usable_as_is_follows_std_for_each_path_shape() { + assert!(usable_as_is(&wide(r"C:\Users\me\data\svc.endpoint-key"))); + assert!(usable_as_is(&wide("C:/Users/me/key"))); + assert!(usable_as_is(&wide("C:"))); + assert!(usable_as_is(&wide(r"\\server\share\key"))); + assert!(usable_as_is(&wide(r"\\?\C:\key"))); + assert!(usable_as_is(&wide(r"\??\C:\key"))); + assert!(!usable_as_is(&wide("relative\\key"))); + assert!(!usable_as_is(&wide(r"\rooted-on-the-current-drive"))); + assert!(!usable_as_is(&wide("C:relative-to-the-drive's-cwd"))); + let long = format!(r"C:\{}", "a".repeat(LEGACY_MAX_PATH)); + assert!(!usable_as_is(&wide(&long))); + // The exact edge: `std` counts the terminating NUL, so a path of 246 + // units (247 with the NUL) still passes as it is and one of 247 does + // not. + let of_len = |len: usize| format!(r"C:\{}", "a".repeat(len - 3)); + assert_eq!(of_len(246).encode_utf16().count(), 246); + assert!(usable_as_is(&wide(&of_len(246)))); + assert!(!usable_as_is(&wide(&of_len(247)))); + assert!(!usable_as_is(&wide(&of_len(248)))); + // A long path that is already verbatim needs nothing. + assert!(usable_as_is(&wide(&format!(r"\\?\{long}")))); + } + + /// Expectations written from `std`'s `get_long_path`, as above. + #[test] + fn with_verbatim_prefix_follows_std_for_each_absolute_shape() { + assert_eq!( + with_verbatim_prefix(&wide(r"C:\Users\me\key")), + wide_nul(r"\\?\C:\Users\me\key") + ); + assert_eq!( + with_verbatim_prefix(&wide(r"\\server\share\key")), + wide_nul(r"\\?\UNC\server\share\key") + ); + assert_eq!( + with_verbatim_prefix(&wide(r"\\.\C:\key")), + wide_nul(r"\\?\C:\key") + ); + assert_eq!( + with_verbatim_prefix(&wide(r"\\?\C:\key")), + wide_nul(r"\\?\C:\key") + ); + assert_eq!( + with_verbatim_prefix(&wide(r"\??\C:\key")), + wide_nul(r"\??\C:\key") + ); + } +} diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index dbe18a856e..8ce4606a14 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -256,6 +256,29 @@ impl TerminationOutcome { ) } + /// Why a stop that is not [`stopped`](Self::stopped) is unconfirmed, worded + /// to complete "process N was not confirmed stopped: …". `None` exactly + /// when [`stopped`](Self::stopped) is `true`, so a warning built from it + /// cannot disagree with the decision it reports. + #[must_use] + pub const fn unconfirmed_reason(self) -> Option<&'static str> { + match self { + Self::TimedOut => { + Some("it, or a process under it, was still running when the stop ended") + } + Self::Unverified => { + Some("its identity could not be confirmed, so it was not signalled") + } + Self::DescendantsUnconfirmed => { + Some("it is gone, but a process under it may still be running") + } + Self::ExitedBeforeTreeKill => Some( + "it exited before the processes under it were stopped, and they may still be running", + ), + Self::AlreadyGone | Self::IdentityMismatch | Self::Graceful | Self::Forced => None, + } + } + /// A stable, log-friendly label for the outcome. #[must_use] pub const fn as_str(self) -> &'static str { @@ -1123,6 +1146,57 @@ mod tests { assert_eq!(parse_start_ticks("123 (short) S 1 2 3"), None); } + #[test] + fn unconfirmed_reason_is_given_exactly_for_an_unconfirmed_stop() { + // The match has no wildcard, so a new variant fails to compile here until + // its reason is pinned; add it to the list below in the same edit. The + // strings pin the wording only. That a warning built from it is printed + // exactly when the stop is unconfirmed, beside the state it describes, is + // proven in the engines by lemonade's + // `state::tests::terminate_pid_reports_an_unconfirmed_stop_on_stderr_and_not_a_confirmed_one` + // and vLLM's + // `process::tests::fail_unready_server_reports_on_stderr_and_kills_a_server_it_could_not_confirm_stopped`. + let expected = |outcome: TerminationOutcome| match outcome { + TerminationOutcome::TimedOut => { + Some("it, or a process under it, was still running when the stop ended") + } + TerminationOutcome::Unverified => { + Some("its identity could not be confirmed, so it was not signalled") + } + TerminationOutcome::DescendantsUnconfirmed => { + Some("it is gone, but a process under it may still be running") + } + TerminationOutcome::ExitedBeforeTreeKill => Some( + "it exited before the processes under it were stopped, and they may still be running", + ), + TerminationOutcome::AlreadyGone + | TerminationOutcome::IdentityMismatch + | TerminationOutcome::Graceful + | TerminationOutcome::Forced => None, + }; + for outcome in [ + TerminationOutcome::AlreadyGone, + TerminationOutcome::IdentityMismatch, + TerminationOutcome::Unverified, + TerminationOutcome::Graceful, + TerminationOutcome::Forced, + TerminationOutcome::TimedOut, + TerminationOutcome::DescendantsUnconfirmed, + TerminationOutcome::ExitedBeforeTreeKill, + ] { + assert_eq!( + outcome.unconfirmed_reason(), + expected(outcome), + "{outcome:?}" + ); + assert_eq!( + outcome.unconfirmed_reason().is_some(), + !outcome.stopped(), + "{outcome:?}" + ); + } + } + #[test] fn outcome_truth_table() { // stopped(): everything except a still-running timeout. diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 5751d17c74..a65ffb91dc 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -182,6 +182,12 @@ pub fn wait_for_process_exit(pid: u32) -> Result { } } +/// Request termination of `pid`: `SIGTERM` on Unix, `TerminateProcess` on Windows. +/// +/// It neither waits for the process to exit nor escalates, so on Unix a process +/// that ignores `SIGTERM` keeps running, and a `wait` on it afterwards can block +/// forever. To stop a process and know it is gone, use +/// [`crate::terminate_verified`]. #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI pub fn terminate_process(pid: u32) -> Result<()> { @@ -207,6 +213,12 @@ pub fn terminate_process(pid: u32) -> Result<()> { Ok(()) } +/// Request termination of `pid`: `SIGTERM` on Unix, `TerminateProcess` on Windows. +/// +/// It neither waits for the process to exit nor escalates, so on Unix a process +/// that ignores `SIGTERM` keeps running, and a `wait` on it afterwards can block +/// forever. To stop a process and know it is gone, use +/// [`crate::terminate_verified`]. #[cfg(not(windows))] #[allow(unsafe_code)] // libc FFI pub fn terminate_process(pid: u32) -> Result<()> { @@ -226,6 +238,11 @@ pub fn terminate_process(pid: u32) -> Result<()> { /// launcher PID leaves those workers reparented to init, where they keep the /// model resident and the device memory pinned. Walking the descendant tree and /// signalling each process avoids that leak. +/// +/// Like [`terminate_process`], this only sends the request: it does not wait +/// for the processes to exit and does not escalate to `SIGKILL`. Use +/// [`crate::terminate_verified`] with [`crate::KillScope::Tree`] to stop a tree +/// and know it is gone. #[cfg(not(windows))] #[allow(unsafe_code)] // libc FFI pub fn terminate_process_tree(pid: u32) -> Result<()> { @@ -348,7 +365,8 @@ fn read_parent_pid(pid: u32) -> Option { /// Terminate `pid` together with every transitive child process. /// /// The Windows implementation falls back to terminating the single process; the -/// engines that rely on descendant cleanup are Unix-only. +/// engines that rely on descendant cleanup are Unix-only. It does not wait for +/// the process to exit; see [`crate::terminate_verified`]. #[cfg(windows)] pub fn terminate_process_tree(pid: u32) -> Result<()> { terminate_process(pid) diff --git a/crates/rocm-core/src/runtime_rocdxg.rs b/crates/rocm-core/src/runtime_rocdxg.rs new file mode 100644 index 0000000000..6f59d5d44f --- /dev/null +++ b/crates/rocm-core/src/runtime_rocdxg.rs @@ -0,0 +1,1100 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which `librocdxg` a managed wheel runtime's HSA runtime actually loads on WSL. +//! +//! The ROCm wheels ship their own `librocdxg.so.1` in `_rocm_sdk_core/lib`, next +//! to `libhsa-runtime64.so.1`. The HSA runtime does not ask for that name: it +//! opens the unversioned `librocdxg.so`, and the wheel ships no such file. The +//! lookup therefore skips the runtime's own copy and lands on whatever the host +//! provides (`/opt/rocm/lib/librocdxg.so` from the `rocdxg-roct` package), which +//! can be older than the runtime expects. A 10.2 or newer nightly runtime asks +//! that copy for `hsaKmtSetPersistingCacheSize`, which `rocdxg-roct` 1.2.2 does +//! not export, and crashes on first GPU use. +//! +//! The mechanism (the unversioned open skipping the runtime's directory and +//! landing on `/opt/rocm/lib`) was observed with `LD_DEBUG=libs` on the 10.2 +//! nightly. Which component requests `hsaKmtSetPersistingCacheSize` is not +//! identified. +//! +//! Linking `librocdxg.so` to the runtime's own `librocdxg.so.1` makes the runtime +//! load the copy it was built with. The link sits in the runtime's own library +//! directory, which `libhsa-runtime64` searches through `$ORIGIN` (a `DT_RPATH` +//! in the 10.1, 10.2 and 10.3 wheels per `readelf -d`, which the loader searches +//! before `LD_LIBRARY_PATH`), so it works for a bare `/bin/python` as +//! well as for a process the CLI launches. +//! +//! The name is judged three ways ([`NameState`]): absent, present and resolving +//! (a regular file or a link that resolves: left alone), and +//! dangling (a link whose target is missing). A dangling `librocdxg.so` is as bad +//! as an absent one: the loader's `open` fails with `ENOENT` on a broken link, +//! tries the next directory and loads the host's copy (`LD_DEBUG=libs` on the +//! 10.2 nightly with a dangling link first on `LD_LIBRARY_PATH` showed the link +//! tried and skipped, then exit 139 as with no link). So detection lists a +//! dangling name like an absent one, and the install replaces it atomically. +//! +//! Limits. The versioned name is the hard-coded `librocdxg.so.1`: a wheel whose +//! soname is no longer that is neither detected nor linked, and no globbing is +//! done (which versioned file would win is a design decision). A dangling +//! `librocdxg.so` beside a runtime that ships no `librocdxg.so.1` is not +//! detected either, because there is nothing to link it to. A versioned copy that +//! is itself dangling (or a link loop) is likewise left out of both `rocm examine` +//! and the install, and deliberately not reported: there is nothing valid to link +//! to, and a runtime whose own `librocdxg.so.1` is broken is a different, wider +//! fault than the one this module covers. +//! +//! "Never replaced" holds for a name that resolves when it is checked. The +//! replacement of a dangling link re-checks the name immediately before the +//! rename, but a name that becomes a working link between that re-check and the +//! rename is still replaced: `rename` has no "only if still dangling" form. The +//! window is short in a single-process, one-shot install (not measured). A name +//! that vanishes between the first check and the re-check is simply linked. +//! +//! Only wheel runtime installs and updates (`install_wheel_runtime`) link. +//! Tarballs ship both names, and an adopted or imported runtime is never linked +//! by the CLI. The link is not tracked by the installer, so an uninstall that +//! removes the runtime's files by package leaves it behind. + +use std::io; +use std::path::{Path, PathBuf}; + +/// The versioned file the wheels ship. +const SHIPPED_NAME: &str = "librocdxg.so.1"; +/// The name the HSA runtime opens. +const OPENED_NAME: &str = "librocdxg.so"; + +/// What is at a path, as the loader sees it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameState { + /// Nothing is there. + Absent, + /// A regular file, or a link whose target exists. + Resolves, + /// A link whose target does not exist, or that loops (`ELOOP`): `open` fails + /// on it, so the loader skips it as it would an absent name. + Dangling, +} + +/// The [`NameState`] of `path` on this machine. +#[must_use] +pub fn name_state(path: &Path) -> NameState { + if path.metadata().is_ok() { + NameState::Resolves + } else if path.symlink_metadata().is_ok() { + NameState::Dangling + } else { + NameState::Absent + } +} + +/// The runtime's own `librocdxg.so.1` for each directory in `dirs` that holds +/// one next to a `libhsa-runtime64` but whose `librocdxg.so` is absent or +/// dangling. +/// +/// In such a directory the runtime's lookup for `librocdxg.so` does not find the +/// runtime's own copy, so it loads the host's. A directory with the library but +/// no runtime beside it is left out: nothing there opens it by name. One whose +/// `librocdxg.so` resolves (a file or a working link) is left out as well; there +/// is nothing for this module to add. A directory whose `librocdxg.so.1` is not +/// there or does not resolve (absent, dangling, or a link loop) is left out: +/// there is nothing valid to link to, and that broken copy is not reported. +/// +/// Takes the state check rather than reading the machine, so the layouts can be +/// tested without one; the real check is [`name_state`]. +#[must_use] +pub fn unlinked_runtime_copies( + dirs: &[PathBuf], + state: impl Fn(&Path) -> NameState, +) -> Vec { + let mut found: Vec = Vec::new(); + for dir in dirs { + let copy = dir.join(SHIPPED_NAME); + let has_runtime = ["libhsa-runtime64.so.1", "libhsa-runtime64.so"] + .iter() + .any(|name| state(&dir.join(name)) == NameState::Resolves); + if state(©) == NameState::Resolves + && state(&dir.join(OPENED_NAME)) != NameState::Resolves + && has_runtime + && !found.contains(©) + { + found.push(copy); + } + } + found +} + +/// The `librocdxg.so.1` copies of every managed runtime whose own HSA runtime +/// would not load them, for `rocm examine`. +/// +/// Empty off WSL (`is_wsl` false): the advice is about the WSL loader path and +/// would mislead on a native host. +#[must_use] +pub fn unlinked_managed_runtime_copies(is_wsl: bool, paths: &crate::AppPaths) -> Vec { + if !is_wsl { + return Vec::new(); + } + unlinked_runtime_copies( + &crate::managed_runtime::managed_runtime_library_dirs(paths), + name_state, + ) +} + +/// The whole post-install step: on WSL, link every runtime copy in `dirs` that +/// its HSA runtime would not load, and return one line per copy saying what +/// happened, for the install report. +/// +/// Reads the WSL gate itself ([`crate::is_wsl_host`], which follows a simulated +/// host root in `e2e-test-hooks` builds), so a caller cannot pass the wrong +/// answer and a test can drive both hosts. +#[must_use] +pub fn link_installed_runtime_copies(dirs: &[PathBuf]) -> Vec { + link_installed_runtime_copies_when(crate::is_wsl_host(), dirs) +} + +/// [`link_installed_runtime_copies`] with the gate supplied. +/// +/// Each line is built from the outcome of that copy's own link attempt, so a +/// link that failed cannot be reported as made. A failure is not an error for +/// the install: the line gives the error and the command to run once its cause +/// is fixed (the same link call failed, so the command alone may fail the same +/// way). Nothing is touched off WSL, where the HSA runtime does not open +/// `librocdxg`. +pub fn link_installed_runtime_copies_when(is_wsl: bool, dirs: &[PathBuf]) -> Vec { + if !is_wsl { + return Vec::new(); + } + link_copies_with(dirs, name_state, link_runtime_copy) +} + +fn link_copies_with( + dirs: &[PathBuf], + state: impl Fn(&Path) -> NameState, + link: impl Fn(&Path) -> io::Result, +) -> Vec { + unlinked_runtime_copies(dirs, state) + .iter() + .map(|copy| match link(copy) { + Ok(linked) if linked.replaced_dangling => format!( + "librocdxg: replaced a dangling {} with a link to the runtime's own copy; the runtime searches its own directory first", + linked.path.display() + ), + Ok(linked) => format!( + "librocdxg: linked {} to the runtime's own copy; the runtime searches its own directory first", + linked.path.display() + ), + Err(error) => format!( + "librocdxg: could not link {}: {error}; the runtime may load the host's librocdxg, which fails at first GPU use if it is older than the runtime expects; once the cause above is fixed, run: {}", + copy.with_file_name(OPENED_NAME).display(), + link_command(copy) + ), + }) + .collect() +} + +/// The command that gives `copy`'s directory the unversioned name, for advice +/// that has to name something the user can run. +/// +/// `-f` and `-n` make it clear a dangling `librocdxg.so` as well as an absent +/// one (a plain `ln -s` fails with "File exists" on a dangling link). The advice +/// is only printed for an absent or dangling name, never for one that resolves. +/// +/// A path containing a newline or invalid UTF-8 is not handled: the advice is one +/// line printed through `Path::display`. +#[must_use] +pub fn link_command(copy: &Path) -> String { + let dir = copy.parent().unwrap_or_else(|| Path::new(".")); + // A single quote in the path is closed, escaped and reopened. + let target = dir + .join(OPENED_NAME) + .display() + .to_string() + .replace('\'', "'\\''"); + format!("ln -sfn {SHIPPED_NAME} '{target}'") +} + +/// What [`link_runtime_copy`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Linked { + /// The `librocdxg.so` link that now exists. + pub path: PathBuf, + /// A dangling `librocdxg.so` was there and was replaced. + pub replaced_dangling: bool, +} + +/// Give `copy`'s directory a `librocdxg.so` that points at `copy`. +/// +/// The link is relative (`librocdxg.so -> librocdxg.so.1`), so it survives the +/// runtime directory being moved or restored from a cache. +/// +/// What is already at `librocdxg.so` decides (see [`NameState`]): nothing, and +/// the link is created; a dangling link, and it is replaced (a temporary link in +/// the same directory is renamed over it, so the name is never absent in +/// between; see `replace_with_link` for the one window that remains); a file +/// or a working link, and it is left alone and reported as an +/// [`io::ErrorKind::AlreadyExists`] error. `copy` itself is not checked: the +/// caller links only a copy that exists. +/// +/// # Errors +/// Returns the underlying error when the link cannot be created, or +/// [`io::ErrorKind::Unsupported`] off Unix (the WSL path is Linux only). +pub fn link_runtime_copy(copy: &Path) -> io::Result { + let dir = copy.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "librocdxg copy has no directory", + ) + })?; + let link = dir.join(OPENED_NAME); + #[cfg(unix)] + { + match name_state(&link) { + NameState::Absent => { + std::os::unix::fs::symlink(SHIPPED_NAME, &link)?; + Ok(Linked { + path: link, + replaced_dangling: false, + }) + } + NameState::Dangling => { + // Whether a dangling link was really replaced is what the final + // check found: a name that vanished since is linked, not replaced. + let replaced_dangling = replace_with_link(dir, &link)?; + Ok(Linked { + path: link, + replaced_dangling, + }) + } + NameState::Resolves => Err(io::Error::new(io::ErrorKind::AlreadyExists, "File exists")), + } + } + #[cfg(not(unix))] + { + let _ = link; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "linking librocdxg is only done on Linux", + )) + } +} + +/// Replace the dangling `link` with a link to [`SHIPPED_NAME`] without a window in +/// which the name is absent: make a temporary link beside it, then rename over it. +/// +/// The name is re-checked just before the rename. Still dangling, or gone since +/// the first check: the temporary is renamed into place (nothing is clobbered). +/// Resolving by now (a file or working link appeared): left alone, with an +/// [`io::ErrorKind::AlreadyExists`] error. That narrows the check-then-act window +/// but cannot close it: a name that becomes a working link between the re-check +/// and the rename is replaced. If the rename fails, or the name is refused, the +/// temporary is removed, and a failure to remove it is part of the returned error. +/// +/// The temporary's name carries the process id and a process-wide counter, so two +/// replacements in one process never share one. A leftover from a crashed process +/// could only collide after the pid is reused, and then `symlink` fails with +/// `File exists`, which is reported, not acted on. +#[cfg(unix)] +fn replace_with_link(dir: &Path, link: &Path) -> io::Result { + replace_with_link_using( + dir, + link, + name_state, + |from, to| std::fs::rename(from, to), + |path| std::fs::remove_file(path), + ) +} + +#[cfg(unix)] +fn replace_with_link_using( + dir: &Path, + link: &Path, + state: impl Fn(&Path) -> NameState, + rename: impl Fn(&Path, &Path) -> io::Result<()>, + remove: impl Fn(&Path) -> io::Result<()>, +) -> io::Result { + use std::sync::atomic::{AtomicU32, Ordering}; + static COUNTER: AtomicU32 = AtomicU32::new(0); + let temporary = dir.join(format!( + ".{OPENED_NAME}.tmp-{}-{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::os::unix::fs::symlink(SHIPPED_NAME, &temporary)?; + // Whether the name dangled at the final check: only then was a link replaced. + let mut replaced_dangling = false; + let outcome = match state(link) { + NameState::Dangling => { + replaced_dangling = true; + rename(&temporary, link) + } + NameState::Absent => rename(&temporary, link), + NameState::Resolves => Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "File exists (it resolves now)", + )), + }; + if let Err(error) = outcome { + return Err(match remove(&temporary) { + Ok(()) => error, + Err(cleanup) => io::Error::new( + error.kind(), + format!( + "{error}; and the temporary link {} could not be removed: {cleanup}", + temporary.display() + ), + ), + }); + } + Ok(replaced_dangling) +} + +#[cfg(test)] +mod tests { + // The tests pin Unix paths and the shell command, and the code under test is + // WSL-only, so they run on Unix only. The module is a plain `cfg(test)` one + // so the registry-writer guard in `apps/rocm/src/runtime_registry.rs` cuts it + // off as test code. + #[cfg(unix)] + mod unix { + use super::super::*; + use std::collections::HashSet; + + fn files(paths: &[&str]) -> impl Fn(&Path) -> NameState { + let set: HashSet = paths.iter().map(PathBuf::from).collect(); + move |path| { + if set.contains(path) { + NameState::Resolves + } else { + NameState::Absent + } + } + } + + fn write(path: &Path, contents: &[u8]) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("dir"); + std::fs::write(path, contents).expect("write"); + } + + #[test] + fn a_runtime_that_ships_only_the_versioned_copy_is_reported() { + let dirs = vec![PathBuf::from("/rt/_rocm_sdk_core/lib")]; + let exists = files(&[ + "/rt/_rocm_sdk_core/lib/librocdxg.so.1", + "/rt/_rocm_sdk_core/lib/libhsa-runtime64.so.1", + ]); + assert_eq!( + unlinked_runtime_copies(&dirs, exists), + vec![PathBuf::from("/rt/_rocm_sdk_core/lib/librocdxg.so.1")] + ); + } + + #[test] + fn a_directory_with_the_unversioned_name_is_not_reported() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = files(&[ + "/rt/lib/librocdxg.so.1", + "/rt/lib/librocdxg.so", + "/rt/lib/libhsa-runtime64.so.1", + ]); + assert!(unlinked_runtime_copies(&dirs, exists).is_empty()); + } + + #[test] + fn the_library_without_a_runtime_beside_it_is_not_reported() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = files(&["/rt/lib/librocdxg.so.1"]); + assert!(unlinked_runtime_copies(&dirs, exists).is_empty()); + } + + #[test] + fn a_runtime_directory_without_the_versioned_copy_is_not_reported() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = files(&["/rt/lib/libhsa-runtime64.so.1"]); + assert!(unlinked_runtime_copies(&dirs, exists).is_empty()); + } + + /// `link_runtime_copy` itself links whatever it is given; the install step + /// is what decides to call it, and it must not make a dangling link in a + /// directory that holds a runtime but no `librocdxg.so.1`. + #[test] + fn the_install_step_makes_no_link_where_the_versioned_copy_is_missing() { + let dir = tempfile::tempdir().expect("temp dir"); + write(&dir.path().join("libhsa-runtime64.so.1"), b""); + let lines = link_installed_runtime_copies_when(true, &[dir.path().to_path_buf()]); + + assert!(lines.is_empty(), "{lines:?}"); + assert!(dir.path().join("librocdxg.so").symlink_metadata().is_err()); + } + + #[test] + fn a_directory_listed_twice_is_reported_once() { + let dirs = vec![PathBuf::from("/rt/lib"), PathBuf::from("/rt/lib")]; + let exists = files(&["/rt/lib/librocdxg.so.1", "/rt/lib/libhsa-runtime64.so"]); + assert_eq!(unlinked_runtime_copies(&dirs, exists).len(), 1); + } + + #[test] + fn link_command_names_the_unversioned_file_beside_the_copy() { + let copy = PathBuf::from("/rt/lib/librocdxg.so.1"); + assert_eq!( + link_command(©), + "ln -sfn librocdxg.so.1 '/rt/lib/librocdxg.so'" + ); + } + + #[test] + fn link_command_escapes_a_single_quote_in_the_path() { + let copy = PathBuf::from("/rt/it's/librocdxg.so.1"); + assert_eq!( + link_command(©), + "ln -sfn librocdxg.so.1 '/rt/it'\\''s/librocdxg.so'" + ); + } + + #[test] + fn the_command_and_the_install_make_the_same_relative_link() { + let by_install = tempfile::tempdir().expect("temp dir"); + let by_command = tempfile::tempdir().expect("temp dir"); + // The command's directory has a quote in it, so the quoting is exercised. + let quoted = by_command.path().join("it's"); + write(&by_install.path().join("librocdxg.so.1"), b"own"); + write("ed.join("librocdxg.so.1"), b"own"); + + let link = link_runtime_copy(&by_install.path().join("librocdxg.so.1")) + .expect("link") + .path; + let status = std::process::Command::new("sh") + .arg("-c") + .arg(link_command("ed.join("librocdxg.so.1"))) + .status() + .expect("run the advised command"); + + assert!(status.success()); + let installed = std::fs::read_link(&link).expect("read_link"); + assert_eq!( + installed, + PathBuf::from("librocdxg.so.1"), + "relative target" + ); + assert_eq!( + std::fs::read_link(quoted.join("librocdxg.so")).expect("read_link"), + installed, + "the advised command makes the link the install makes" + ); + } + + #[test] + fn a_failed_link_is_reported_as_failed_with_the_manual_command() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = files(&["/rt/lib/librocdxg.so.1", "/rt/lib/libhsa-runtime64.so.1"]); + let lines = link_copies_with(&dirs, exists, |_| { + Err(io::Error::new(io::ErrorKind::PermissionDenied, "denied")) + }); + assert_eq!(lines.len(), 1); + assert!(lines[0].contains("could not link"), "{lines:?}"); + assert!(!lines[0].contains("linked /"), "{lines:?}"); + assert!( + lines[0].contains("ln -sfn librocdxg.so.1 '/rt/lib/librocdxg.so'"), + "{lines:?}" + ); + } + + /// A runtime directory whose `librocdxg.so` is a link to nowhere. + fn runtime_dir_with_a_dangling_link() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("temp dir"); + write(&dir.path().join("librocdxg.so.1"), b"own"); + write(&dir.path().join("libhsa-runtime64.so.1"), b""); + std::os::unix::fs::symlink("gone.so.9", dir.path().join("librocdxg.so")) + .expect("dangling link"); + dir + } + + /// The runtime directory with the versioned name in a given state; the + /// unversioned name is absent. + fn runtime_dir_with_versioned_name(make: impl Fn(&Path)) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("temp dir"); + write(&dir.path().join("libhsa-runtime64.so.1"), b""); + make(&dir.path().join("librocdxg.so.1")); + dir + } + + fn temporaries_in(dir: &Path) -> usize { + std::fs::read_dir(dir) + .expect("read_dir") + .flatten() + .filter(|entry| entry.file_name().to_string_lossy().contains(".tmp-")) + .count() + } + + #[test] + fn the_versioned_copy_decides_whether_a_directory_is_listed() { + type Make = fn(&Path); + let cases: [(&str, Make, bool); 5] = [ + ("a regular file", |p| write(p, b"own"), true), + ( + "a link that resolves", + |p| { + write(&p.with_file_name("real.so"), b"own"); + std::os::unix::fs::symlink("real.so", p).expect("link"); + }, + true, + ), + ("absent", |_| {}, false), + ( + "a dangling link", + |p| std::os::unix::fs::symlink("gone.so", p).expect("link"), + false, + ), + ( + "a link loop", + |p| std::os::unix::fs::symlink("librocdxg.so.1", p).expect("link"), + false, + ), + ]; + for (what, make, listed) in cases { + let dir = runtime_dir_with_versioned_name(make); + let dirs = vec![dir.path().to_path_buf()]; + let found = unlinked_runtime_copies(&dirs, name_state); + assert_eq!(found.is_empty(), !listed, "{what}: {found:?}"); + + let lines = link_installed_runtime_copies_when(true, &dirs); + assert_eq!(lines.is_empty(), !listed, "{what}: {lines:?}"); + assert_eq!( + dir.path().join("librocdxg.so").symlink_metadata().is_ok(), + listed, + "{what}: a link is made only for a copy that can be linked to" + ); + assert_eq!(temporaries_in(dir.path()), 0, "{what}"); + } + } + + #[test] + fn examine_does_not_list_a_runtime_whose_versioned_copy_dangles() { + let (root, paths) = crate::test_support::temp_app_paths("runtime-rocdxg-dangling-copy"); + let lib = root.join("wheel/lib"); + write(&lib.join("libhsa-runtime64.so.1"), b""); + std::os::unix::fs::symlink("gone.so", lib.join("librocdxg.so.1")).expect("link"); + register_runtime( + &paths.data_dir.join("runtimes").join("registry"), + "k", + &[&lib], + ); + + assert!(unlinked_managed_runtime_copies(true, &paths).is_empty()); + } + + #[test] + fn examine_scans_every_directory_the_install_would_link() { + // The install links the probe's `library_paths`; the registry record + // holds the same list, and examine scans what the environment builder + // makes of it. Both are computed separately, so pin that the second + // contains the first for a wheel-shaped runtime. + let (root, paths) = crate::test_support::temp_app_paths("runtime-rocdxg-dir-sets"); + let wheel = root.join("rt/lib/python3.14/site-packages"); + let probe_library_paths = [ + wheel.join("_rocm_sdk_core/lib"), + wheel.join("_rocm_sdk_core/lib/rocm_sysdeps/lib"), + wheel.join("_rocm_sdk_libraries/lib"), + ]; + for dir in &probe_library_paths { + std::fs::create_dir_all(dir).expect("dir"); + } + let refs: Vec<&Path> = probe_library_paths.iter().map(PathBuf::as_path).collect(); + register_runtime( + &paths.data_dir.join("runtimes").join("registry"), + "k", + &refs, + ); + + let scanned = crate::managed_runtime::managed_runtime_library_dirs(&paths); + for dir in &probe_library_paths { + assert!( + scanned.contains(dir), + "examine does not scan {dir:?}: {scanned:?}" + ); + } + } + + #[test] + fn a_dangling_unversioned_link_is_listed_like_an_absent_one() { + let dir = runtime_dir_with_a_dangling_link(); + let dirs = vec![dir.path().to_path_buf()]; + + assert_eq!( + name_state(&dir.path().join("librocdxg.so")), + NameState::Dangling + ); + assert_eq!( + unlinked_runtime_copies(&dirs, name_state), + vec![dir.path().join("librocdxg.so.1")], + "the loader skips a broken link, so the runtime loads the host's copy" + ); + } + + #[test] + fn the_install_step_replaces_a_dangling_link_and_leaves_no_temporary() { + let dir = runtime_dir_with_a_dangling_link(); + let lines = link_installed_runtime_copies_when(true, &[dir.path().to_path_buf()]); + + let link = dir.path().join("librocdxg.so"); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].contains(&format!("replaced a dangling {}", link.display())), + "{lines:?}" + ); + assert_eq!(std::fs::read(&link).expect("resolves now"), b"own"); + assert_eq!( + std::fs::read_link(&link).expect("read_link"), + PathBuf::from("librocdxg.so.1") + ); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .expect("read_dir") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".tmp-")) + .collect(); + assert!( + leftovers.is_empty(), + "no temporary link is left: {leftovers:?}" + ); + } + + #[test] + fn a_resolving_link_or_a_file_is_never_replaced() { + for make in [ + |dir: &Path| { + write(&dir.join("other.so"), b"theirs"); + std::os::unix::fs::symlink("other.so", dir.join("librocdxg.so")).expect("link"); + }, + |dir: &Path| write(&dir.join("librocdxg.so"), b"theirs"), + ] { + let dir = runtime_dir_shipping_only_the_versioned_copy(); + make(dir.path()); + let lines = link_installed_runtime_copies_when(true, &[dir.path().to_path_buf()]); + + assert!(lines.is_empty(), "nothing to do: {lines:?}"); + assert_eq!( + std::fs::read(dir.path().join("librocdxg.so")).expect("untouched"), + b"theirs" + ); + assert!(link_runtime_copy(&dir.path().join("librocdxg.so.1")).is_err()); + assert_eq!( + std::fs::read(dir.path().join("librocdxg.so")).expect("still untouched"), + b"theirs" + ); + } + } + + #[test] + fn a_name_that_stopped_dangling_before_the_rename_is_left_alone() { + let dir = runtime_dir_shipping_only_the_versioned_copy(); + write(&dir.path().join("librocdxg.so"), b"theirs"); + let link = dir.path().join("librocdxg.so"); + // The caller saw a dangling name; by the re-check it resolves. + let outcome = replace_with_link(dir.path(), &link); + + assert_eq!( + outcome.expect_err("refused").kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(std::fs::read(&link).expect("untouched"), b"theirs"); + let leftovers = std::fs::read_dir(dir.path()) + .expect("read_dir") + .flatten() + .filter(|entry| entry.file_name().to_string_lossy().contains(".tmp-")) + .count(); + assert_eq!(leftovers, 0, "the temporary link is removed"); + } + + #[test] + fn a_failed_cleanup_is_part_of_the_error() { + let dir = runtime_dir_with_a_dangling_link(); + let link = dir.path().join("librocdxg.so"); + let outcome = replace_with_link_using( + dir.path(), + &link, + name_state, + |_, _| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "rename denied", + )) + }, + |_| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "remove denied", + )) + }, + ); + + let message = outcome.expect_err("failed").to_string(); + assert!(message.contains("rename denied"), "{message}"); + assert!( + message.contains("and the temporary link") + && message.contains("could not be removed: remove denied"), + "{message}" + ); + } + + #[test] + fn a_failed_rename_removes_the_temporary_link_from_disk() { + let dir = runtime_dir_with_a_dangling_link(); + let link = dir.path().join("librocdxg.so"); + // The rename is injected to fail (the name still dangles at the + // re-check, so the rename is reached); removal is the real one, so the + // temporary link that was made on disk must be gone afterwards. + let outcome = replace_with_link_using( + dir.path(), + &link, + name_state, + |_, _| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "rename denied", + )) + }, + |path| std::fs::remove_file(path), + ); + + assert!(outcome.is_err()); + assert_eq!( + temporaries_in(dir.path()), + 0, + "the temporary link is removed" + ); + assert_eq!( + name_state(&link), + NameState::Dangling, + "the dangling name is as it was" + ); + } + + #[test] + fn a_name_that_vanished_before_the_recheck_is_linked() { + let dir = runtime_dir_shipping_only_the_versioned_copy(); + let link = dir.path().join("librocdxg.so"); + // The caller saw a dangling name; by the re-check nothing is there. + let outcome = replace_with_link_using( + dir.path(), + &link, + |_| NameState::Absent, + |from, to| std::fs::rename(from, to), + |path| std::fs::remove_file(path), + ); + + // Nothing was replaced: the name was gone, so `false`. + assert!(!outcome.expect("linked")); + assert_eq!(std::fs::read(&link).expect("resolves"), b"own"); + assert_eq!(temporaries_in(dir.path()), 0); + } + + #[test] + fn the_three_cases_each_print_their_own_line() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = || files(&["/rt/lib/librocdxg.so.1", "/rt/lib/libhsa-runtime64.so.1"]); + let line = |replaced_dangling| { + link_copies_with(&dirs, exists(), |copy| { + Ok(Linked { + path: copy.with_file_name("librocdxg.so"), + replaced_dangling, + }) + }) + .remove(0) + }; + + // Absent from the start, and vanished at the re-check, both `false`. + assert!(line(false).starts_with("librocdxg: linked /rt/lib/librocdxg.so to")); + assert!(!line(false).contains("replaced")); + assert!( + line(true).starts_with("librocdxg: replaced a dangling /rt/lib/librocdxg.so with") + ); + } + + #[test] + fn a_dangling_name_replaced_reports_true_and_an_absent_one_linked_reports_false() { + let dir = runtime_dir_with_a_dangling_link(); + let copy = dir.path().join("librocdxg.so.1"); + let replaced = link_runtime_copy(©).expect("replaced"); + assert!(replaced.replaced_dangling); + + let absent = runtime_dir_shipping_only_the_versioned_copy(); + let linked = link_runtime_copy(&absent.path().join("librocdxg.so.1")).expect("linked"); + assert!(!linked.replaced_dangling); + } + + /// Two threads replacing dangling links in one directory at once (two + /// runtime copies in one directory is the shape the counter exists for): + /// every temporary name must differ, or one rename fails or a link is lost. + #[test] + fn concurrent_replacements_in_one_directory_use_distinct_temporary_names() { + const ROUNDS: usize = 200; + let dir = tempfile::tempdir().expect("temp dir"); + let barrier = std::sync::Barrier::new(2); + let results: Vec>> = std::thread::scope(|scope| { + let run = |name: &'static str| { + let (dir, barrier) = (dir.path(), &barrier); + scope.spawn(move || { + (0..ROUNDS) + .map(|round| { + let link = dir.join(format!("{name}-{round}")); + std::os::unix::fs::symlink("gone", &link).expect("dangling"); + barrier.wait(); + replace_with_link(dir, &link) + }) + .collect::>() + }) + }; + let (a, b) = (run("a"), run("b")); + vec![a.join().expect("thread a"), b.join().expect("thread b")] + }); + + for outcome in results.iter().flatten() { + assert!(outcome.is_ok(), "{outcome:?}"); + } + for name in ["a", "b"] { + for round in 0..ROUNDS { + let link = dir.path().join(format!("{name}-{round}")); + assert_eq!( + std::fs::read_link(&link).expect("still a link"), + PathBuf::from("librocdxg.so.1"), + "{link:?}" + ); + } + } + } + + #[test] + fn the_advised_command_clears_a_dangling_link() { + let dir = runtime_dir_with_a_dangling_link(); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(link_command(&dir.path().join("librocdxg.so.1"))) + .status() + .expect("run the advised command"); + + assert!( + status.success(), + "a plain `ln -s` would fail with File exists" + ); + assert_eq!( + std::fs::read(dir.path().join("librocdxg.so")).expect("resolves"), + b"own" + ); + assert!(unlinked_runtime_copies(&[dir.path().to_path_buf()], name_state).is_empty()); + } + + fn runtime_dir_shipping_only_the_versioned_copy() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("temp dir"); + write(&dir.path().join("librocdxg.so.1"), b"own"); + write(&dir.path().join("libhsa-runtime64.so.1"), b""); + dir + } + + #[test] + fn the_install_step_on_wsl_links_and_reports_it() { + let dir = runtime_dir_shipping_only_the_versioned_copy(); + let lines = link_installed_runtime_copies_when(true, &[dir.path().to_path_buf()]); + + let link = dir.path().join("librocdxg.so"); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].contains(&format!( + "linked {} to the runtime's own copy", + link.display() + )), + "{lines:?}" + ); + assert_eq!(std::fs::read(&link).expect("through link"), b"own"); + } + + #[test] + fn the_install_step_off_wsl_touches_nothing() { + let dir = runtime_dir_shipping_only_the_versioned_copy(); + let lines = link_installed_runtime_copies_when(false, &[dir.path().to_path_buf()]); + + assert!(lines.is_empty(), "{lines:?}"); + assert!( + dir.path().join("librocdxg.so").symlink_metadata().is_err(), + "no link may be made off WSL" + ); + } + + /// Drives the real gate, [`crate::is_wsl_host`], through the simulated host + /// root, so the entry point the install calls is tested on a WSL host and on a + /// native one. Needs the `e2e-test-hooks` feature (see docs/testing.md). + #[cfg(feature = "e2e-test-hooks")] + #[test] + fn the_install_entry_point_reads_the_wsl_gate_from_the_host() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let wsl_host = tempfile::tempdir().expect("temp dir"); + write(&wsl_host.path().join("dev/dxg"), b""); + let native_host = tempfile::tempdir().expect("temp dir"); + write( + &native_host.path().join("proc/version"), + b"Linux version 6.8.0-generic", + ); + + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + let native = runtime_dir_shipping_only_the_versioned_copy(); + let root_guard = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, native_host.path()); + let lines = link_installed_runtime_copies(&[native.path().to_path_buf()]); + assert!(lines.is_empty(), "{lines:?}"); + assert!( + native + .path() + .join("librocdxg.so") + .symlink_metadata() + .is_err() + ); + drop(root_guard); + + let wsl = runtime_dir_shipping_only_the_versioned_copy(); + let _wsl_root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, wsl_host.path()); + let lines = link_installed_runtime_copies(&[wsl.path().to_path_buf()]); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert_eq!( + std::fs::read(wsl.path().join("librocdxg.so")).expect("through link"), + b"own" + ); + } + + #[test] + fn the_install_step_reports_a_failed_link_with_the_manual_command() { + // The failure does not rely on permissions (root ignores them): the state + // check handed to `link_copies_with` is told `librocdxg.so` is absent + // while a file really is there, so `link_runtime_copy`'s own check of + // the name refuses it with `AlreadyExists` ("File exists") for any + // user. It is that check, not the OS `symlink` call, that fails here. + let dir = runtime_dir_shipping_only_the_versioned_copy(); + write(&dir.path().join("librocdxg.so"), b"someone else's file"); + let absent_unversioned = |path: &Path| { + if path.file_name().is_some_and(|name| name == "librocdxg.so") { + NameState::Absent + } else { + name_state(path) + } + }; + let lines = link_copies_with( + &[dir.path().to_path_buf()], + absent_unversioned, + link_runtime_copy, + ); + + assert_eq!(lines.len(), 1, "{lines:?}"); + let link = dir.path().join("librocdxg.so"); + // The error text stays in the line, and the command is offered only + // once its cause is fixed: the same link call failed here. + assert!( + lines[0].contains(&format!("could not link {}: ", link.display())) + && lines[0].contains("File exists"), + "{lines:?}" + ); + assert!( + lines[0].contains("once the cause above is fixed, run: ln -sfn librocdxg.so.1"), + "{lines:?}" + ); + assert_eq!( + std::fs::read(dir.path().join("librocdxg.so")).expect("untouched"), + b"someone else's file" + ); + } + + fn register_runtime(registry: &Path, key: &str, libs: &[&Path]) { + let record = serde_json::json!({ + "runtime_key": key, "runtime_id": "therock-nightly:gfx1151", + "rocm_sdk": { "import_ok": true, "library_paths": libs }, + }); + write( + ®istry.join(format!("{key}.json")), + record.to_string().as_bytes(), + ); + } + + #[test] + fn examine_lists_the_copies_of_every_registered_runtime_only_on_wsl() { + let (root, paths) = crate::test_support::temp_app_paths("runtime-rocdxg-registry"); + let (first, shared, second) = ( + root.join("first/lib"), + root.join("shared/lib"), + root.join("second/lib"), + ); + for lib in [&first, &shared, &second] { + write(&lib.join("librocdxg.so.1"), b""); + write(&lib.join("libhsa-runtime64.so.1"), b""); + } + let registry = paths.data_dir.join("runtimes").join("registry"); + // Two records, one directory in both: every runtime is scanned, and the + // shared directory is listed once. + register_runtime(®istry, "a", &[&first, &shared]); + register_runtime(®istry, "b", &[&shared, &second]); + + let mut listed = unlinked_managed_runtime_copies(true, &paths); + listed.sort(); + let mut expected = vec![ + first.join("librocdxg.so.1"), + shared.join("librocdxg.so.1"), + second.join("librocdxg.so.1"), + ]; + expected.sort(); + assert_eq!(listed, expected); + assert!( + unlinked_managed_runtime_copies(false, &paths).is_empty(), + "a native host gets no WSL advice" + ); + } + + #[test] + fn a_made_link_is_reported_as_made() { + let dirs = vec![PathBuf::from("/rt/lib")]; + let exists = files(&["/rt/lib/librocdxg.so.1", "/rt/lib/libhsa-runtime64.so.1"]); + let lines = link_copies_with(&dirs, exists, |copy| { + Ok(Linked { + path: copy.with_file_name("librocdxg.so"), + replaced_dangling: false, + }) + }); + assert_eq!(lines.len(), 1); + assert!( + lines[0].contains("linked /rt/lib/librocdxg.so to the runtime's own copy"), + "{lines:?}" + ); + } + + #[test] + fn linking_makes_the_unversioned_name_resolve_to_the_runtimes_copy() { + let dir = tempfile::tempdir().expect("temp dir"); + let copy = dir.path().join("librocdxg.so.1"); + write(©, b"runtime copy"); + write(&dir.path().join("libhsa-runtime64.so.1"), b""); + let dirs = vec![dir.path().to_path_buf()]; + assert_eq!(unlinked_runtime_copies(&dirs, name_state).len(), 1); + + let link = link_runtime_copy(©).expect("link").path; + + assert_eq!(link, dir.path().join("librocdxg.so")); + assert_eq!( + std::fs::read_link(&link).expect("read_link"), + PathBuf::from("librocdxg.so.1"), + "the target is relative, so the link survives the directory moving" + ); + assert_eq!( + std::fs::read(&link).expect("read through link"), + b"runtime copy" + ); + assert!( + unlinked_runtime_copies(&dirs, name_state).is_empty(), + "once linked, the runtime is no longer reported" + ); + assert!( + link_runtime_copy(©).is_err(), + "a librocdxg.so that resolves is left alone" + ); + } + } +} diff --git a/crates/rocm-core/src/tensorizer_override.rs b/crates/rocm-core/src/tensorizer_override.rs new file mode 100644 index 0000000000..498f0ba7d6 --- /dev/null +++ b/crates/rocm-core/src/tensorizer_override.rs @@ -0,0 +1,368 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The tensorizer release rocm-cli installs over vLLM's own exact pin, and the +//! judgement that a `uv pip check` finding about it is deliberate. +//! +//! vLLM's wheel declares `tensorizer==2.10.1`, and that release does not import +//! under Python 3.14 (its libsodium-constants metaclass reads class annotations +//! the way Python 3.13 and earlier stored them). ROCm 10.1.0's vLLM is published +//! for Python 3.14 only, so the engine install upgrades tensorizer to +//! [`TENSORIZER_PYTHON314_FIX`] when the pinned release cannot be imported. The +//! exact pin then reads as unmet in `uv pip check`, which both the vLLM engine +//! (to decide whether to reinstall) and the CLI (to report `dependency_check`) +//! must tell apart from a defect; they share the rule here. + +use crate::ViolationSubject; + +/// The first tensorizer release that imports under Python 3.14 (coreweave/tensorizer#204). +pub const TENSORIZER_PYTHON314_FIX: &str = "2.12.1"; + +/// The package whose pin rocm-cli deliberately overrides. +pub const TENSORIZER_PACKAGE: &str = "tensorizer"; + +/// `uv pip install` requirement for the override. +pub fn tensorizer_override_requirement() -> String { + format!("{TENSORIZER_PACKAGE}=={TENSORIZER_PYTHON314_FIX}") +} + +/// The numeric release of a plain version, or `None` for anything else. +/// +/// Accepted: dotted components that are runs of ASCII digits, then optionally one +/// last `post` component (a post-release counts as the release it follows), +/// and then optionally one `+local` segment made only of ASCII letters, digits and +/// dots with no empty component (`+rocm10.1.0.g98dff2a81.d20261002`, `+x`). The +/// local segment is checked but not read. +/// Everything else has no release and fails closed: a pre-release (`2.12.1rc1`, +/// `dev0`), a bare `post`, text after the `post` component, a marker or +/// other trailing text, `-`, spaces, non-ASCII digits, a component too large for +/// `u64`, an empty or repeated `+` segment, and the empty string. +/// +/// Leading zeros read as the number (`2.10.01` is 2.10.1). Trailing zero components +/// are kept: a longer release compares greater than its prefix, which already gives +/// the right answer for the one release this is compared with (`2.12.1.0` is not +/// below 2.12.1, `2.10.1.0` is, and `2.12` is) without normalising them away. +fn release(version: &str) -> Option> { + let (public, local) = match version.split_once('+') { + Some((public, local)) => (public, Some(local)), + None => (version, None), + }; + if let Some(local) = local + && !local + .split('.') + .all(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_alphanumeric())) + { + // Empty, a second `+`, or anything but letters, digits and dots. + return None; + } + let mut parts = Vec::new(); + let mut components = public.split('.').peekable(); + while let Some(part) = components.next() { + if let Some(rest) = part.strip_prefix("post") + && !parts.is_empty() + && components.peek().is_none() + && !rest.is_empty() + && rest.bytes().all(|byte| byte.is_ascii_digit()) + { + // `post` as the last component: the release it follows. + break; + } + // `u64::parse` is the digit guard: it rejects an empty component, a sign + // (`-1`; a `+` cannot reach here, the string was split on it), non-ASCII + // digits, anything non-numeric and an overflow. + parts.push(part.parse::().ok()?); + } + (!parts.is_empty()).then_some(parts) +} + +/// Whether `version` is at least [`TENSORIZER_PYTHON314_FIX`]. +/// +/// Compares the dotted numeric release. A version that has no plain release is +/// not treated as fixed. +pub fn tensorizer_version_is_fixed(version: &str) -> bool { + match (release(version), release(TENSORIZER_PYTHON314_FIX)) { + (Some(have), Some(need)) => have >= need, + _ => false, + } +} + +/// Whether `version` is a plain release strictly below the fix. Unlike +/// `!tensorizer_version_is_fixed`, a version that cannot be read is not below it. +fn tensorizer_version_is_below_fix(version: &str) -> bool { + match (release(version), release(TENSORIZER_PYTHON314_FIX)) { + (Some(have), Some(need)) => have < need, + _ => false, + } +} + +/// Whether this unmet requirement is the tensorizer override. +/// +/// `violation_subject` fills `required` only for a lone `==` pin, as the text after +/// the `==`. That text must read as a plain release strictly below the fix (the +/// pin that cannot be imported); a pin that does not parse (garbage, empty, or +/// qualified by an environment marker, which leaves trailing text) stays a +/// violation, as does a pin on a release at or above the fix. The installed +/// release must be at least the fix. A tensorizer that is missing or older is a +/// real defect. +pub fn is_tensorizer_override(subject: &ViolationSubject) -> bool { + subject.package.eq_ignore_ascii_case(TENSORIZER_PACKAGE) + && subject + .required + .as_deref() + .is_some_and(tensorizer_version_is_below_fix) + && subject + .installed + .as_deref() + .is_some_and(tensorizer_version_is_fixed) +} + +/// The tensorizer release installed when `detail` is vLLM's own unmet tensorizer +/// pin that [`is_tensorizer_override`] accepts. +/// +/// `detail` is one `uv pip check` line. Only vLLM's requirement qualifies: another +/// package's unmet tensorizer pin is not something the vLLM install put there. +/// Returning the installed release lets a caller print the version the runtime +/// holds instead of the constant it was upgraded to. +pub fn vllm_tensorizer_override_installed(detail: &str) -> Option { + let (requirer, _) = detail.split_once(crate::uv::UV_CHECK_REQUIREMENT_INFIX)?; + if !requirer.contains("`vllm`") { + return None; + } + let subject = crate::violation_subject(detail)?; + if !is_tensorizer_override(&subject) { + return None; + } + subject.installed +} + +#[cfg(test)] +mod tests { + use super::*; + + fn subject(package: &str, installed: Option<&str>) -> ViolationSubject { + ViolationSubject { + package: package.to_owned(), + required: Some("2.10.1".to_owned()), + installed: installed.map(str::to_owned), + } + } + + #[test] + fn versions_at_or_above_the_fix_count_as_fixed() { + assert!(tensorizer_version_is_fixed("2.12.1")); + assert!(tensorizer_version_is_fixed("2.12.2")); + assert!(tensorizer_version_is_fixed("2.13.0")); + assert!(tensorizer_version_is_fixed("3.0")); + assert!(!tensorizer_version_is_fixed("2.12.0")); + assert!(!tensorizer_version_is_fixed("2.10.1")); + assert!(!tensorizer_version_is_fixed("2.9.9")); + assert!(!tensorizer_version_is_fixed("garbage")); + } + + #[test] + fn only_a_fixed_tensorizer_is_the_override() { + assert!(is_tensorizer_override(&subject( + "tensorizer", + Some("2.12.1") + ))); + assert!(is_tensorizer_override(&subject( + "Tensorizer", + Some("2.12.1") + ))); + assert!(!is_tensorizer_override(&subject( + "tensorizer", + Some("2.9.0") + ))); + assert!(!is_tensorizer_override(&subject("tensorizer", None))); + assert!(!is_tensorizer_override(&subject("torch", Some("2.12.1")))); + // A pin on a release that is not the broken one is unmet for another reason. + let mut newer_pin = subject("tensorizer", Some("2.12.1")); + newer_pin.required = Some("2.13.0".to_owned()); + assert!(!is_tensorizer_override(&newer_pin)); + let mut loose_pin = subject("tensorizer", Some("2.12.1")); + loose_pin.required = None; + assert!(!is_tensorizer_override(&loose_pin)); + // A pin that cannot be read must not pass as the deliberate override. + for required in [ + "garbage", + "", + "2.10.1; python_version >= \"3.14\"", + "2.10.1rc1", + "2.12.1", + "2.12.1+local", + ] { + let mut pin = subject("tensorizer", Some("2.12.1")); + pin.required = Some(required.to_owned()); + assert!( + !is_tensorizer_override(&pin), + "{required:?} must stay a violation" + ); + } + // The plain pin still counts, with a local segment too. + for required in ["2.10.1", "2.11.0", "2.10.1+local"] { + let mut pin = subject("tensorizer", Some("2.12.1")); + pin.required = Some(required.to_owned()); + assert!(is_tensorizer_override(&pin), "{required:?}"); + } + } + + #[test] + fn a_prerelease_of_the_fix_is_not_fixed() { + assert!(!tensorizer_version_is_fixed("2.12.1rc1")); + assert!(!tensorizer_version_is_fixed("2.12.1.dev0")); + assert!(tensorizer_version_is_fixed("2.12.1+local")); + // Only a `+` local segment is dropped; anything else after the release is + // not a plain release. + assert!(!tensorizer_version_is_fixed("2.12.1-1")); + assert!(!tensorizer_version_is_fixed("2.12.1 (x)")); + // A post-release of the fixed release is at least that release. + assert!(tensorizer_version_is_fixed("2.12.1.post1")); + assert!(!tensorizer_version_is_fixed("2.12.0.post1")); + } + + #[test] + fn only_vllms_own_unmet_pin_is_reported_with_the_installed_release() { + let pin = |requirer: &str, installed: &str| { + format!( + "The package `{requirer}` requires `tensorizer==2.10.1`, but `{installed}` is installed" + ) + }; + assert_eq!( + vllm_tensorizer_override_installed(&pin("vllm", "2.12.1")).as_deref(), + Some("2.12.1") + ); + assert_eq!( + vllm_tensorizer_override_installed(&pin("vllm", "2.13.0")).as_deref(), + Some("2.13.0") + ); + assert_eq!( + vllm_tensorizer_override_installed(&pin("lemonade", "2.12.1")), + None + ); + assert_eq!( + vllm_tensorizer_override_installed(&pin("vllm", "2.9.0")), + None + ); + } + + #[test] + fn a_post_release_is_read_only_in_its_plain_shape() { + // Accepted: post as the last component, with or without +local. + assert!(tensorizer_version_is_fixed("2.12.1.post1")); + assert!(tensorizer_version_is_fixed("2.12.1.post0+local")); + assert!(!tensorizer_version_is_fixed("2.12.0.post1")); + let pin = |required: &str| { + let mut pin = subject("tensorizer", Some("2.12.1")); + pin.required = Some(required.to_owned()); + pin + }; + assert!(is_tensorizer_override(&pin("2.10.1.post1"))); + assert!(is_tensorizer_override(&pin("2.10.1.post1+local"))); + // Anything else around `post` fails closed: the pin stays a violation and + // an installed version of that shape is not fixed. + for text in [ + "2.10.1.postgarbage", + "2.10.1.post1.foo", + "2.10.1.post", + "2.10.1.post1x", + "2.10.1.post1.post2", + "2.10.1.post-1", + "post1", + ] { + assert!(!is_tensorizer_override(&pin(text)), "{text:?} pin"); + let installed = text.replace("2.10.1", "2.12.1"); + assert!( + !tensorizer_version_is_fixed(&installed), + "{installed:?} installed" + ); + } + } + + #[test] + fn spellings_of_one_release_compare_equal_and_unreadable_ones_fail_closed() { + // Trailing zero components and leading zeros do not change the verdict. + assert!(tensorizer_version_is_fixed("2.12.1.0")); + assert!(tensorizer_version_is_fixed("2.12.01")); + assert!(!tensorizer_version_is_fixed("2.12.0.0")); + assert!(!tensorizer_version_is_fixed("2.12")); + assert!(tensorizer_version_is_below_fix("2.10.1.0")); + assert!(tensorizer_version_is_below_fix("2.10.01")); + assert!(tensorizer_version_is_below_fix("2.12")); + assert!(!tensorizer_version_is_below_fix("2.12.1.0")); + assert!(!tensorizer_version_is_below_fix("2.12.01")); + // Unreadable: neither fixed nor below the fix. + for text in [ + "", + "2..1", + ".2.1", + "2.10.1-1", + "2.10.1 ", + "2.10.1", + "2.10.99999999999999999999999", + ] { + assert!(!tensorizer_version_is_fixed(text), "{text:?} fixed"); + assert!(!tensorizer_version_is_below_fix(text), "{text:?} below"); + } + } + + #[test] + fn the_requirement_frame_is_the_one_uv_check_parsing_uses() { + // Built from the shared constant, not a literal, so a change to the frame + // `violation_subject` splits on is a change here too. + let detail = format!( + "The package `vllm`{}tensorizer==2.10.1`, but `2.12.1` is installed", + crate::uv::UV_CHECK_REQUIREMENT_INFIX + ); + assert_eq!( + vllm_tensorizer_override_installed(&detail).as_deref(), + Some("2.12.1") + ); + } + + #[test] + fn a_local_segment_is_read_only_in_its_plain_shape() { + let pin = |required: &str| { + let mut pin = subject("tensorizer", Some("2.12.1")); + pin.required = Some(required.to_owned()); + pin + }; + // Real local segments, and the short ones, still read. + for local in ["+rocm10.1.0.g98dff2a81.d20261002", "+local", "+x", "+1"] { + assert!( + is_tensorizer_override(&pin(&format!("2.10.1{local}"))), + "pin {local}" + ); + assert!( + tensorizer_version_is_fixed(&format!("2.12.1{local}")), + "installed {local}" + ); + } + // Malformed ones do not: the pin stays a violation, the installed version + // is not fixed. + for local in [ + "+", + "+a+b", + "+a..b", + "+.a", + "+a.", + "+a-b", + "+a b", + "+\u{ff11}", + ] { + assert!( + !is_tensorizer_override(&pin(&format!("2.10.1{local}"))), + "pin {local:?}" + ); + assert!( + !tensorizer_version_is_fixed(&format!("2.12.1{local}")), + "installed {local:?}" + ); + } + } + + #[test] + fn the_override_requirement_names_the_fix() { + assert_eq!(tensorizer_override_requirement(), "tensorizer==2.12.1"); + } +} diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index e89a0d1121..fd33ed9655 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -254,7 +254,7 @@ const UV_CHECK_VIOLATION_PREFIX: &str = "The package `"; /// `Requires-Python`, a package with multiple installed distributions — and none of them /// is a `Requires-Dist` a reinstall of the requirer can resolve. Only a requirement /// `uv` quotes is one. -const UV_CHECK_REQUIREMENT_INFIX: &str = " requires `"; +pub(crate) const UV_CHECK_REQUIREMENT_INFIX: &str = " requires `"; /// Report the unsatisfied `Requires-Dist` entries in the environment at `venv_python`. /// diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 97f9784571..edfe6974d3 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -1092,6 +1092,73 @@ mod tests { } } + /// The bin's `validate_chat_tool_call` refuses any ROCm tool call that + /// carries an `arguments` field, so the approval card and the approved + /// command cannot read different payloads. That is only safe while no ROCm + /// tool here declares a parameter of that name. + #[tokio::test] + async fn no_rocm_tool_declares_an_arguments_parameter() { + let fired: FiredLog = Arc::new(Mutex::new(Vec::new())); + macro_rules! read_def { + ($ty:ident) => { + $ty { + executor: None, + fired: fired.clone(), + } + .definition(String::new()) + .await + }; + } + macro_rules! mutating_def { + ($ty:ident) => { + $ty { + executor: None, + approval_tx: None, + fired: fired.clone(), + } + .definition(String::new()) + .await + }; + } + let defs = [ + read_def!(DoctorRocmTool), + read_def!(ExamineRocmTool), + read_def!(EnginesRocmTool), + read_def!(ServicesRocmTool), + read_def!(ServiceLogsRocmTool), + read_def!(BridgeSnapshotRocmTool), + read_def!(GpuSnapshotRocmTool), + read_def!(AutomationsRocmTool), + read_def!(PathExistsRocmTool), + read_def!(PortStatusRocmTool), + read_def!(UpdateCheckRocmTool), + read_def!(InstallSdkDryRunRocmTool), + read_def!(RocmCommandRocmTool), + read_def!(NaturalLanguagePlanRocmTool), + mutating_def!(InstallSdkRocmTool), + mutating_def!(InstallEngineRocmTool), + mutating_def!(LaunchServerRocmTool), + mutating_def!(StopServerRocmTool), + mutating_def!(WatcherEnableRocmTool), + mutating_def!(WatcherDisableRocmTool), + ]; + assert_eq!( + defs.len(), + ROCM_READ_TOOL_NAMES.len() + ROCM_MUTATING_TOOL_NAMES.len(), + "every ROCm tool is checked" + ); + for def in defs { + assert!( + def.parameters + .get("properties") + .and_then(|p| p.get("arguments")) + .is_none(), + "ROCm tool `{}` declares an `arguments` parameter", + def.name + ); + } + } + /// Minimal executor that echoes a fixed JSON value for any tool call. #[derive(Debug)] struct FakeExec(serde_json::Value); diff --git a/crates/rocm-dash-tui/src/ui/approval.rs b/crates/rocm-dash-tui/src/ui/approval.rs index 6fa82ffef9..2e33a06ffa 100644 --- a/crates/rocm-dash-tui/src/ui/approval.rs +++ b/crates/rocm-dash-tui/src/ui/approval.rs @@ -323,6 +323,17 @@ mod tests { (text, max) } + #[test] + fn an_approval_dialog_shows_its_review_title_and_approve_button() { + // The e2e step `no_approval_prompt_displayed` (dash_steps.rs) proves a + // dialog is absent by these two strings never appearing on screen, so + // they are pinned here, where the dialog draws them: renaming either + // must fail this test rather than let that step pass vacuously. + let (out, _) = draw(&long_request(3), None); + assert!(out.contains("Review: "), "{out}"); + assert!(out.contains("Approve (y)"), "{out}"); + } + #[test] fn a_plain_approval_never_promises_scroll_keys() { // Only the callers that handle PgUp and PgDn opt in. Every other dialog diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index ee95efbf0d..281e36272e 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -17,14 +17,14 @@ include!(concat!(env!("OUT_DIR"), "/pins.rs")); /// GitHub repository publishing the Lemonade embeddable release archives. pub const LEMONADE_GITHUB_REPO: &str = "lemonade-sdk/lemonade"; -/// Strip a leading `v` from a release tag (`v11.5.1` -> `11.5.1`). +/// Strip a leading `v` from a release tag (`v2026.44.1` -> `2026.44.1`). #[must_use] pub fn strip_v(tag: &str) -> &str { tag.strip_prefix('v').unwrap_or(tag) } /// Name of the Lemonade embeddable archive for a host token and extension, -/// e.g. `lemonade-embeddable-11.5.1-ubuntu-x64.tar.gz`. +/// e.g. `lemonade-embeddable-2026.44.1-ubuntu-x64.tar.gz`. #[must_use] pub fn lemonade_archive_name(version: &str, os_arch: &str, extension: &str) -> String { format!( diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index cdb00339fa..31af67a769 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -11,8 +11,10 @@ publish.workspace = true workspace = true [features] -# Compiles `fake_engine`, a stand-in engine child for tests, into the `rocm` and -# `rocmd` test builds. Never enabled in a release build. +# Compiles the test-only modules into the test builds that turn it on (`rocm`, +# `rocmd`, and the `lemonade`, `vllm` and `aim` engines): `fake_engine`, a +# stand-in engine child (Unix only), and `engine_contract`, the shared engine +# protocol contract check. Never enabled in a release build. test-support = [] [dependencies] @@ -22,6 +24,9 @@ serde.workspace = true serde_json.workspace = true [dev-dependencies] +# The leftover a killed owner-only key store leaves, for the tests of the +# cleanups that remove it (`atomic_write::stage_owner_only_file_unpublished`). +rocm-core = { path = "../rocm-core", features = ["test-support"] } # Property tests for the engine protocol decoders (`tests/protocol_properties.rs`). proptest = { version = "1", default-features = false, features = ["std"] } diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs index dc90a42449..d8ed29d004 100644 --- a/crates/rocm-engine-protocol/src/fake_engine.rs +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -5,13 +5,15 @@ //! A stand-in engine child for tests. //! //! It is a `/bin/sh` script that records how it was launched (arguments, -//! environment, stdin) into a scratch directory and then runs a caller-supplied +//! environment, stdin, and the environment and request of each of its runs) into a +//! scratch directory and then runs a caller-supplied //! body, so a test can assert what an engine IPC client put on the child's //! command line, environment and stdin, and shape what comes back (a response //! envelope, a crash, stderr output). //! -//! Test-only: compiled for this crate's own tests and for the `rocm` and -//! `rocmd` test builds through the `test-support` feature, so the two engine +//! Test-only: compiled for this crate's own tests and for every test build +//! that turns on the `test-support` feature (`rocm`, `rocmd`, and the three +//! engine crates, which turn it on for `engine_contract`), so the two engine //! IPC callers are exercised against the same fake rather than each growing a //! private one. Unix-only, because the fake is a shell script; the pure parts //! of the client are tested on every platform without it. @@ -59,14 +61,14 @@ impl FakeEngine { fs::create_dir_all(&dir).expect("create the fake engine's scratch directory"); let script = dir.join("engine.sh"); let read_stdin = if reads_stdin { - "cat > \"$dir/stdin\"\n" + "cat > \"$dir/stdin\"\ncat \"$dir/stdin\" >> \"$dir/requests\"\n" } else { "" }; rocm_core::fake_executable::write_fake_executable( &script, format!( - "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\nrun=0\nwhile [ -e \"$dir/env.$run\" ]; do run=$((run + 1)); done\nenv > \"$dir/env.$run\"\nprintf '%s\\n' \"$@\" > \"$dir/argv.$run\"\n{read_stdin}{body}\n", shell_quote(&dir.to_string_lossy()) ), ) @@ -88,25 +90,79 @@ impl FakeEngine { &self.script } - /// The arguments the fake was started with, one per entry. + /// The arguments the fake was started with the last time it was run, one + /// per entry. pub fn argv(&self) -> Vec { - fs::read_to_string(self.dir.join("argv")) - .expect("the fake engine never recorded its arguments") + Self::lines_of(&self.dir.join("argv")) + } + + /// The arguments the fake was started with the `run`-th time it was run + /// (0 is the first), one per entry. + pub fn argv_of_run(&self, run: usize) -> Vec { + Self::lines_of(&self.dir.join(format!("argv.{run}"))) + } + + fn lines_of(file: &Path) -> Vec { + fs::read_to_string(file) + .unwrap_or_else(|error| { + panic!( + "the fake engine recorded no arguments at {}: {error}", + file.display() + ) + }) .lines() .map(str::to_owned) .collect() } - /// Everything the fake read from its stdin, verbatim. + /// Every request the fake read, one per time it was run, oldest first. A + /// fake that was never run (or never read its stdin) has an empty list. + pub fn requests(&self) -> Vec { + fs::read_to_string(self.dir.join("requests")) + .unwrap_or_default() + .lines() + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|error| panic!("a request is not JSON ({error}): {line:?}")) + }) + .collect() + } + + /// The `method` of every request the fake read, oldest first. + pub fn methods(&self) -> Vec { + self.requests() + .iter() + .map(|request| request["method"].as_str().unwrap_or_default().to_owned()) + .collect() + } + + /// Everything the fake read from its stdin the last time it was run, + /// verbatim. pub fn stdin(&self) -> String { fs::read_to_string(self.dir.join("stdin")).expect("the fake engine never read its stdin") } - /// The value of `key` in the fake's environment, if it was set. + /// The value of `key` in the fake's environment the last time it was run, + /// if it was set. pub fn env(&self, key: &str) -> Option { + Self::env_in(&self.dir.join("env"), key) + } + + /// The value of `key` in the fake's environment the `run`-th time it was + /// started (0 is the first), if it was set. + pub fn env_of_run(&self, run: usize, key: &str) -> Option { + Self::env_in(&self.dir.join(format!("env.{run}")), key) + } + + fn env_in(file: &Path, key: &str) -> Option { let prefix = format!("{key}="); - fs::read_to_string(self.dir.join("env")) - .expect("the fake engine never recorded its environment") + fs::read_to_string(file) + .unwrap_or_else(|error| { + panic!( + "the fake engine recorded no environment at {}: {error}", + file.display() + ) + }) .lines() .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) } diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6b3f4065bf..9fcab53800 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -5,6 +5,14 @@ #[cfg(all(unix, any(test, feature = "test-support")))] pub mod fake_engine; +mod protocol_version; +#[cfg(any(test, feature = "test-support"))] +pub use protocol_version::engine_contract; +pub use protocol_version::{ + INCOMPATIBLE_PROTOCOL_CODE, INVALID_PAYLOAD_CODE, INVALID_REQUEST_CODE, + PRE_VERSIONING_PROTOCOL, ProtocolMismatch, UNKNOWN_METHOD_CODE, answer_engine_request, + engine_protocol_compatible, incompatible_request_response, +}; mod stdio_client; pub use stdio_client::{ ENGINE_PROGRESS_STDERR_ENV, EngineStdioOptions, decode_engine_response, @@ -18,12 +26,17 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +/// The version of the engine stdio protocol this build speaks. +/// +/// It is carried in every request and answer as `protocol_version`. See the +/// `protocol_version` module for what a missing field means, the compatibility +/// rule and when to raise it. pub const ENGINE_PROTOCOL_VERSION: &str = "0.1.0"; pub const ENGINE_RECIPE_CONTRACT_VERSION: &str = "0.1.0"; pub const ENGINE_PLUGIN_BINARY_PREFIX: &str = "rocm-engine-"; pub const DEFAULT_LOG_TAIL_LINES: usize = 80; -/// Environment variable holding a *path* to a 0600 file whose contents are the +/// Environment variable holding a *path* to an owner-only file whose contents are the /// endpoint API key. /// /// `rocm serve` sets this explicitly on the internal `__engine-serve-http` child @@ -110,7 +123,7 @@ pub fn endpoint_api_key_from_file(path: &std::path::Path) -> Option { } } -/// Deterministic path of the 0600 endpoint key file for `service_id`. +/// Deterministic path of the owner-only endpoint key file for `service_id`. /// /// Shared so both binaries that need to locate a service's key file — `rocm` /// (which owns writing and clearing it) and `rocmd` (which only re-threads @@ -152,6 +165,25 @@ pub fn endpoint_key_file_if_present( path.exists().then_some(path) } +/// Remove `service_id`'s endpoint key file and any leftover staged copy of it. +/// +/// A store killed between staging and publishing leaves a staged copy beside +/// the key (see `rocm_core::atomic_write::staged_siblings`), and this removes +/// the copies it finds along with the key. Best-effort and idempotent: a +/// missing file is not an error, a file it cannot remove is left where it is, +/// and if the folder cannot be listed the staged copies are left, all without +/// being reported, because no caller can act on a failed cleanup. +/// +/// It removes the key and the staged copies present when the folder is +/// listed. It does not exclude a store for the same service running at the +/// same time, which may still publish a key or may fail. +pub fn remove_endpoint_key_files(paths: &rocm_core::AppPaths, service_id: &str) { + let key = endpoint_key_file_path(paths, service_id); + for path in std::iter::once(key.clone()).chain(rocm_core::atomic_write::staged_siblings(&key)) { + let _ = fs::remove_file(&path); + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct EnginePluginDescriptor { pub id: String, @@ -250,6 +282,61 @@ pub enum EngineMethod { Logs, } +impl EngineMethod { + /// Every method, in declaration order. `every_method_is_in_all` holds this + /// list to the enum's derived decoder, so a new method that is not added + /// here fails a test rather than slipping past every loop over it. + pub const ALL: [Self; 9] = [ + Self::Detect, + Self::Install, + Self::Capabilities, + Self::ResolveModel, + Self::Launch, + Self::Healthcheck, + Self::Endpoint, + Self::Stop, + Self::Logs, + ]; + + /// The method's name on the wire (`install`, `resolve_model`, ...). + #[must_use] + pub const fn name(&self) -> &'static str { + match self { + Self::Detect => "detect", + Self::Install => "install", + Self::Capabilities => "capabilities", + Self::ResolveModel => "resolve_model", + Self::Launch => "launch", + Self::Healthcheck => "healthcheck", + Self::Endpoint => "endpoint", + Self::Stop => "stop", + Self::Logs => "logs", + } + } + + /// Whether the client checks the engine's protocol version before sending + /// this request: the methods that install or start something (a runtime, a + /// server), which must never reach an engine that would misread them. The + /// others are answered in the engine's protocol and that answer's version is + /// checked instead. + /// + /// `stop` changes state too, and is left out on purpose: it starts no extra + /// child for a check first, so a stop is never held behind one. An engine + /// that checks the version refuses a `stop` made for a protocol it cannot + /// serve, before reading its payload. An engine from before the field ignores + /// the version and acts on the `stop`; its answer names no version and reads + /// as `0.1.0`, so the client accepts it while this release's protocol is + /// `0.1.x` and refuses it once the protocol is past that. When the engine's + /// answer is an error or is refused, the stop paths in `rocm` and `rocmd` + /// still signal the server's recorded processes themselves; only for an + /// engine whose stop alone can confirm it (`aim`) does that leave the stop + /// unconfirmed. + #[must_use] + pub const fn requires_protocol_handshake(&self) -> bool { + matches!(self, Self::Install | Self::Launch) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum DevicePolicy { @@ -343,11 +430,28 @@ pub fn launch_gpu_indices(selection: Option<&GpuSelection>) -> Vec { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EngineRequestEnvelope { + /// The protocol the request was written in; absent means + /// [`PRE_VERSIONING_PROTOCOL`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub protocol_version: Option, pub method: EngineMethod, #[serde(default)] pub payload: Value, } +impl EngineRequestEnvelope { + /// A request for `method` with `payload`, written in this build's + /// [`ENGINE_PROTOCOL_VERSION`]. + #[must_use] + pub fn new(method: EngineMethod, payload: Value) -> Self { + Self { + protocol_version: Some(ENGINE_PROTOCOL_VERSION.to_owned()), + method, + payload, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EngineErrorDetail { pub code: String, @@ -356,6 +460,10 @@ pub struct EngineErrorDetail { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EngineResponseEnvelope { + /// The protocol the answer was written in; absent means + /// [`PRE_VERSIONING_PROTOCOL`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub protocol_version: Option, pub ok: bool, #[serde(skip_serializing_if = "Option::is_none")] pub data: Option, @@ -369,6 +477,7 @@ impl EngineResponseEnvelope { T: Serialize, { Self { + protocol_version: Some(ENGINE_PROTOCOL_VERSION.to_owned()), ok: true, data: Some(serde_json::to_value(data).expect("serializable response")), error: None, @@ -377,6 +486,7 @@ impl EngineResponseEnvelope { pub fn failure(code: impl Into, message: impl Into) -> Self { Self { + protocol_version: Some(ENGINE_PROTOCOL_VERSION.to_owned()), ok: false, data: None, error: Some(EngineErrorDetail { @@ -818,6 +928,31 @@ mod tests { std::fs::remove_dir_all(&paths.data_dir).ok(); } + /// A store killed between staging and publishing leaves a staged copy of + /// the key; removing the service's key takes that copy too, and leaves + /// another service's key and staged copy alone. + #[test] + fn remove_endpoint_key_files_takes_the_key_and_its_leftover_staged_copies() { + let paths = test_app_paths("key-file-remove"); + let key = endpoint_key_file_path(&paths, "svc-a"); + std::fs::create_dir_all(key.parent().unwrap()).unwrap(); + std::fs::write(&key, "published").unwrap(); + let staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&key, b"staged").unwrap(); + let other_key = endpoint_key_file_path(&paths, "svc-ab"); + let other_staged = + rocm_core::atomic_write::stage_owner_only_file_unpublished(&other_key, b"other") + .unwrap(); + + remove_endpoint_key_files(&paths, "svc-a"); + let left = (key.exists(), staged.exists(), other_staged.exists()); + // A second removal finds nothing and is a no-op. + remove_endpoint_key_files(&paths, "svc-a"); + std::fs::remove_dir_all(&paths.data_dir).ok(); + + assert_eq!(left, (false, false, true)); + } + #[test] fn endpoint_key_file_if_present_reflects_existence_only() { let paths = test_app_paths("key-file-if-present"); @@ -865,14 +1000,131 @@ mod tests { #[test] fn request_roundtrip_preserves_method() { - let envelope = EngineRequestEnvelope { - method: EngineMethod::Detect, - payload: serde_json::json!({ "runtime_id": "therock-release" }), - }; + let envelope = EngineRequestEnvelope::new( + EngineMethod::Detect, + serde_json::json!({ "runtime_id": "therock-release" }), + ); let serialized = serde_json::to_string(&envelope).unwrap(); let parsed: EngineRequestEnvelope = serde_json::from_str(&serialized).unwrap(); assert_eq!(parsed.method, EngineMethod::Detect); + assert_eq!( + parsed.protocol_version.as_deref(), + Some(ENGINE_PROTOCOL_VERSION) + ); + } + + /// Every message this build writes names the protocol it was written in, + /// and a message from before the field existed still reads, with no version. + #[test] + fn envelopes_carry_this_builds_protocol_version_and_tolerate_its_absence() { + let request = serde_json::to_value(EngineRequestEnvelope::new( + EngineMethod::Stop, + serde_json::json!({}), + )) + .unwrap(); + assert_eq!(request["protocol_version"], ENGINE_PROTOCOL_VERSION); + let success = serde_json::to_value(EngineResponseEnvelope::success(1)).unwrap(); + assert_eq!(success["protocol_version"], ENGINE_PROTOCOL_VERSION); + let failure = serde_json::to_value(EngineResponseEnvelope::failure("c", "m")).unwrap(); + assert_eq!(failure["protocol_version"], ENGINE_PROTOCOL_VERSION); + + let legacy_request: EngineRequestEnvelope = + serde_json::from_str(r#"{"method":"stop","payload":{}}"#).unwrap(); + assert_eq!(legacy_request.protocol_version, None); + let legacy_response: EngineResponseEnvelope = + serde_json::from_str(r#"{"ok":true,"data":{}}"#).unwrap(); + assert_eq!(legacy_response.protocol_version, None); + + // And a message with no version is written without the field, as one + // from before it was. + let mut request = EngineRequestEnvelope::new(EngineMethod::Stop, serde_json::json!({})); + request.protocol_version = None; + let request = serde_json::to_value(request).unwrap(); + assert!(request.get("protocol_version").is_none(), "{request}"); + let mut response = EngineResponseEnvelope::success(1); + response.protocol_version = None; + let response = serde_json::to_value(response).unwrap(); + assert!(response.get("protocol_version").is_none(), "{response}"); + } + + /// `EngineMethod::ALL` lists every variant, checked against the enum's own + /// derive: decoding a name that is not a method fails with the full list of + /// method names serde knows, so a variant added to the enum and not to + /// `ALL` fails here. + #[test] + fn every_method_is_in_all() { + let error = serde_json::from_value::(serde_json::json!("not-a-method")) + .unwrap_err() + .to_string(); + let known: Vec<&str> = error + .split_once("expected one of ") + .unwrap_or_else(|| panic!("unexpected serde error: {error}")) + .1 + .split(", ") + .map(|name| name.trim_matches('`')) + .collect(); + let listed: Vec<&str> = EngineMethod::ALL.iter().map(EngineMethod::name).collect(); + assert_eq!(listed, known); + } + + /// Each method is put on one side of the protocol check by hand: the match + /// is exhaustive, so a new method does not compile here until someone + /// decides whether the client checks the engine's protocol before sending it. + #[test] + fn every_method_is_placed_on_one_side_of_the_protocol_check() { + for method in EngineMethod::ALL { + let checked_first = match method { + EngineMethod::Install | EngineMethod::Launch => true, + EngineMethod::Detect + | EngineMethod::Capabilities + | EngineMethod::ResolveModel + | EngineMethod::Healthcheck + | EngineMethod::Endpoint + | EngineMethod::Stop + | EngineMethod::Logs => false, + }; + assert_eq!( + method.requires_protocol_handshake(), + checked_first, + "{method:?}" + ); + } + } + + #[test] + fn error_codes_have_their_wire_spellings() { + assert_eq!(INCOMPATIBLE_PROTOCOL_CODE, "incompatible_protocol"); + assert_eq!(UNKNOWN_METHOD_CODE, "unknown_method"); + assert_eq!(INVALID_REQUEST_CODE, "invalid_request"); + assert_eq!(INVALID_PAYLOAD_CODE, "invalid_payload"); + } + + /// A field this build does not know, as a newer protocol on the same minor + /// may add, is ignored in both envelopes rather than refused. + #[test] + fn envelopes_ignore_a_field_they_do_not_know() { + let request: EngineRequestEnvelope = serde_json::from_str( + r#"{"protocol_version":"0.1.9","method":"stop","payload":{},"hint":"new"}"#, + ) + .unwrap(); + assert_eq!(request.method, EngineMethod::Stop); + let response: EngineResponseEnvelope = serde_json::from_str( + r#"{"protocol_version":"0.1.9","ok":true,"data":{"stopped":true},"hint":"new"}"#, + ) + .unwrap(); + assert!(response.ok); + assert_eq!(response.data, Some(serde_json::json!({"stopped": true}))); + } + + #[test] + fn method_names_match_the_wire_spelling() { + for method in EngineMethod::ALL { + assert_eq!( + serde_json::to_value(&method).unwrap(), + serde_json::json!(method.name()) + ); + } } #[test] diff --git a/crates/rocm-engine-protocol/src/protocol_version.rs b/crates/rocm-engine-protocol/src/protocol_version.rs new file mode 100644 index 0000000000..e706663077 --- /dev/null +++ b/crates/rocm-engine-protocol/src/protocol_version.rs @@ -0,0 +1,775 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which engine protocol a request or an answer was written in, and whether +//! the other side can use it. +//! +//! Every request and every answer carries `protocol_version`, the +//! [`ENGINE_PROTOCOL_VERSION`] of the side that wrote it, as a +//! `MAJOR.MINOR.PATCH` string. A message without one was written before the +//! field existed and is read as [`PRE_VERSIONING_PROTOCOL`] (`0.1.0`), so an +//! engine or a daemon from an earlier release keeps working with this one. +//! +//! Two versions are compatible by the caret rule: while the major is `0`, the +//! major and the minor must both match (`0.1.0` and `0.1.9` are, `0.1.0` and +//! `0.2.0` are not); from `1.0.0` on, the major alone must match. The patch +//! never decides, in either direction. A `MAJOR.MINOR.PATCH` version here is +//! three dot-separated plain decimal numbers, each without a leading zero +//! (`0` itself aside) and fitting in 64 bits. Anything else cannot be compared +//! with anything, and is treated as incompatible and reported as unreadable +//! rather than as a different version. +//! +//! [`ENGINE_PROTOCOL_VERSION`] `0.1.0` is the baseline: it is the release that +//! introduced the field, and it speaks the same format as the messages written +//! before it, which is why a missing field and `0.1.0` mean the same thing. +//! +//! Rules for raising [`ENGINE_PROTOCOL_VERSION`] from there. They are this +//! protocol's own: the recipe hint's `ENGINE_RECIPE_CONTRACT_VERSION` is +//! compared as an exact string, and the `rocm fix` catalog keeps its number on +//! an addition, so neither is the model here. +//! - An **addition** an older peer can ignore — a new optional field, a new +//! method only an engine ever offers — raises the patch, which every peer on +//! the same minor (major, from `1.0.0`) still accepts. +//! - A **new method a client may send** raises the minor while the major is +//! `0`, and the major after that: an older engine cannot decode it and would +//! answer `unknown_method` (see [`answer_engine_request`]). The exception is a +//! method the client sends only to an engine whose capabilities say it has it. +//! - A **removal or a type change** — a field dropped, renamed, made required or +//! given another type — raises the minor while the major is `0`, and the major +//! from `1.0.0` on, so that a peer that would misread the message refuses it +//! instead of guessing. +//! +//! An engine reads a request's version before anything else in it, its method +//! included, and refuses a request it cannot serve with an error answer whose +//! code is [`INCOMPATIBLE_PROTOCOL_CODE`] ([`answer_engine_request`] for a +//! request read from stdin, [`incompatible_request_response`] for one already +//! decoded). The client checks the version of an answer that is a single JSON +//! object before anything else in it, and before an `install` or a `launch` +//! it first asks the engine for its capabilities and checks the version of +//! that answer, so a request that installs or starts something is never sent +//! to an engine that cannot read it. + +use serde_json::Value; + +use crate::{ENGINE_PROTOCOL_VERSION, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope}; + +/// The version a request or an answer without `protocol_version` was written +/// in: the format from before the field existed. +pub const PRE_VERSIONING_PROTOCOL: &str = "0.1.0"; + +/// The error code an engine answers with when a request was made for a +/// protocol it cannot serve. +pub const INCOMPATIBLE_PROTOCOL_CODE: &str = "incompatible_protocol"; + +/// The error code an engine answers with when a request in a protocol it can +/// serve names a method it does not have. +pub const UNKNOWN_METHOD_CODE: &str = "unknown_method"; + +/// The error code an engine answers with when what it was sent is not a +/// request envelope at all. +pub const INVALID_REQUEST_CODE: &str = "invalid_request"; + +/// The error code an engine answers with when a request's payload does not +/// decode as that method's request. +pub const INVALID_PAYLOAD_CODE: &str = "invalid_payload"; + +/// Why two engine protocol versions cannot be used together. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProtocolMismatch { + /// Both versions are readable, and the caret rule says they differ. + Incompatible { ours: String, theirs: String }, + /// The other side's version is not a `MAJOR.MINOR.PATCH` version. + Unreadable { ours: String, theirs: String }, +} + +/// `version` as its three numbers, or `None` when it is not exactly +/// `MAJOR.MINOR.PATCH` as the module documentation defines it. +fn parse_version(version: &str) -> Option<(u64, u64, u64)> { + let mut parts = version.split('.'); + let mut number = || { + let part = parts.next()?; + if part.is_empty() + || !part.bytes().all(|byte| byte.is_ascii_digit()) + || (part.len() > 1 && part.starts_with('0')) + { + return None; + } + part.parse::().ok() + }; + let parsed = (number()?, number()?, number()?); + if parts.next().is_some() { + return None; + } + Some(parsed) +} + +/// Whether a side that speaks `ours` can use a peer that speaks `theirs`. +/// +/// The rule is the caret rule in the module documentation. `None` is a message +/// written before the field existed, read as [`PRE_VERSIONING_PROTOCOL`]. +/// +/// `ours` is this build's own constant; should it ever fail to parse, nothing +/// is compatible with it, which a test of [`ENGINE_PROTOCOL_VERSION`] rules out. +pub fn engine_protocol_compatible( + ours: &str, + theirs: Option<&str>, +) -> Result<(), ProtocolMismatch> { + let theirs = theirs.unwrap_or(PRE_VERSIONING_PROTOCOL); + let Some(their_version) = parse_version(theirs) else { + return Err(ProtocolMismatch::Unreadable { + ours: ours.to_owned(), + theirs: theirs.to_owned(), + }); + }; + let compatible = parse_version(ours).is_some_and(|our_version| { + if our_version.0 == 0 || their_version.0 == 0 { + our_version.0 == their_version.0 && our_version.1 == their_version.1 + } else { + our_version.0 == their_version.0 + } + }); + if compatible { + Ok(()) + } else { + Err(ProtocolMismatch::Incompatible { + ours: ours.to_owned(), + theirs: theirs.to_owned(), + }) + } +} + +/// The answer an engine gives instead of serving `request` when the request +/// was made for a protocol this build cannot serve, or `None` when it can. +/// +/// Each engine calls this before it decodes the payload, so a request whose +/// payload means something else in another protocol is never acted on. A +/// request read from text goes through [`answer_engine_request`], which judges +/// the version before the method is even decoded. +#[must_use] +pub fn incompatible_request_response( + request: &EngineRequestEnvelope, +) -> Option { + refusal_for(request.protocol_version.as_deref()) +} + +/// The `incompatible_protocol` answer to a request that named `theirs`, or +/// `None` when this build can serve it. +fn refusal_for(theirs: Option<&str>) -> Option { + let mismatch = engine_protocol_compatible(ENGINE_PROTOCOL_VERSION, theirs).err()?; + let message = match &mismatch { + ProtocolMismatch::Incompatible { ours, theirs } => format!( + "this engine speaks engine protocol {ours}; the request was made for protocol \ + {theirs}, which it cannot serve" + ), + ProtocolMismatch::Unreadable { ours, theirs } => format!( + "this engine speaks engine protocol {ours}; the request named protocol version \ + {theirs:?}, which is not a MAJOR.MINOR.PATCH version, so it cannot be served" + ), + }; + Some(EngineResponseEnvelope::failure( + INCOMPATIBLE_PROTOCOL_CODE, + message, + )) +} + +/// The `protocol_version` a JSON object names, read without decoding the +/// rest of it: `None` when it is absent or `null`, the string itself, or, for +/// any other type, its JSON text (which is never a readable version). Both +/// sides use it to judge a message's version before decoding the message. +pub(crate) fn raw_protocol_version(value: &Value) -> Option { + match value.get("protocol_version") { + None | Some(Value::Null) => None, + Some(Value::String(version)) => Some(version.clone()), + Some(other) => Some(other.to_string()), + } +} + +/// Answer the request an engine read from its stdin, as `text`: judge its +/// `protocol_version` first, then decode the rest and hand it to `handle`. +/// +/// The version is read from the request as plain JSON before the envelope is +/// decoded, so a request made for another protocol is refused with +/// [`INCOMPATIBLE_PROTOCOL_CODE`] whatever its method is (a later protocol may +/// add methods this build cannot decode) and whatever type its version has (a +/// version that is present and neither a string nor `null` is unreadable, like +/// one that is not `MAJOR.MINOR.PATCH`; `null` is read like a missing version, +/// as [`PRE_VERSIONING_PROTOCOL`]). A request this build can serve that names a +/// method it does not have is answered with [`UNKNOWN_METHOD_CODE`], and text +/// that is not a request envelope at all, including JSON that is not an object, +/// with [`INVALID_REQUEST_CODE`]. Every outcome is an answer envelope. Input +/// that cannot be read as text (not valid UTF-8) never reaches this function: +/// the engine's stdio command fails reading it, and there is no answer. +pub fn answer_engine_request( + text: &str, + handle: impl FnOnce(EngineRequestEnvelope) -> EngineResponseEnvelope, +) -> EngineResponseEnvelope { + let value: Value = match serde_json::from_str(text) { + Ok(value) => value, + Err(error) => { + return EngineResponseEnvelope::failure( + INVALID_REQUEST_CODE, + format!("the request is not JSON: {error}"), + ); + } + }; + // A derived struct also decodes from a JSON array of its fields in order, + // which no client writes; an array has no `protocol_version` to judge, so + // reading one by position would serve a request whose version was never + // checked. + if !value.is_object() { + return EngineResponseEnvelope::failure( + INVALID_REQUEST_CODE, + "the request is not an engine request envelope: it is not a JSON object", + ); + } + let version = raw_protocol_version(&value); + if let Some(refusal) = refusal_for(version.as_deref()) { + return refusal; + } + if let Some(Value::String(method)) = value.get("method") + && serde_json::from_value::(Value::String(method.clone())).is_err() + { + return EngineResponseEnvelope::failure( + UNKNOWN_METHOD_CODE, + format!( + "this engine (engine protocol {ENGINE_PROTOCOL_VERSION}) has no method {method:?}" + ), + ); + } + match serde_json::from_value::(value) { + Ok(request) => handle(request), + Err(error) => EngineResponseEnvelope::failure( + INVALID_REQUEST_CODE, + format!("the request is not an engine request envelope: {error}"), + ), + } +} + +/// The protocol contract every engine holds, as one shared check. +/// +/// Each engine's tests run it, so the three first-party engines (and `rocm`'s +/// built-in stdio verb) are held to the same list rather than to three +/// hand-copied ones. +#[cfg(any(test, feature = "test-support"))] +pub mod engine_contract { + use super::{ + ENGINE_PROTOCOL_VERSION, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, + INCOMPATIBLE_PROTOCOL_CODE, INVALID_PAYLOAD_CODE, INVALID_REQUEST_CODE, + UNKNOWN_METHOD_CODE, + }; + + fn code(answer: &EngineResponseEnvelope) -> Option<&str> { + answer.error.as_ref().map(|error| error.code.as_str()) + } + + /// The first version the documented bump rule makes incompatible with + /// `version`: the next minor while the major is `0`, the next major after. + pub(super) fn breaking_successor(version: &str) -> String { + let (major, minor, _) = + super::parse_version(version).expect("a protocol version is MAJOR.MINOR.PATCH"); + if major == 0 { + format!("0.{}.0", minor + 1) + } else { + format!("{}.0.0", major + 1) + } + } + + #[track_caller] + fn assert_refused(engine: &str, request: &str, answer: &EngineResponseEnvelope, code_: &str) { + assert!(!answer.ok, "{engine}: {request} was served"); + assert_eq!( + code(answer), + Some(code_), + "{engine}: {request}: {:?}", + answer.error + ); + assert_eq!( + answer.protocol_version.as_deref(), + Some(ENGINE_PROTOCOL_VERSION), + "{engine}: {request}: the answer must name this build's protocol" + ); + } + + /// Check `engine` against the contract, through `answer_text` (what its + /// stdio command does with the text it read) and `handle` (its decoded + /// envelope handler, which `rocm` calls in-process): + /// - a request for an incompatible or unreadable protocol is refused with + /// `incompatible_protocol` for every method on both paths, and, on the + /// stdio path, for a method this build does not know and for a version + /// that is not a string; + /// - a compatible request naming an unknown method is `unknown_method`, and + /// text that is not a request envelope is `invalid_request`; + /// - `capabilities` in this protocol, and with no version at all, is served, + /// and an `install` or `launch` with an empty payload reaches the engine's + /// own decoding (`invalid_payload`), with this build's version on every + /// answer. + #[track_caller] + pub fn assert_engine_holds_protocol_contract( + engine: &str, + answer_text: impl Fn(&str) -> EngineResponseEnvelope, + handle: impl Fn(EngineRequestEnvelope) -> EngineResponseEnvelope, + ) { + let breaking = breaking_successor(ENGINE_PROTOCOL_VERSION); + let breaking_text = format!("{breaking:?}"); + for theirs in [breaking_text.as_str(), r#""banana""#, "2", "[0,1,0]"] { + for method in EngineMethod::ALL + .iter() + .map(EngineMethod::name) + .chain(["frobnicate"]) + { + let request = format!( + r#"{{"protocol_version":{theirs},"method":"{method}","payload":{{}}}}"# + ); + assert_refused( + engine, + &request, + &answer_text(&request), + INCOMPATIBLE_PROTOCOL_CODE, + ); + } + } + for method in EngineMethod::ALL { + for theirs in [breaking.as_str(), "banana"] { + let mut request = EngineRequestEnvelope::new(method.clone(), serde_json::json!({})); + request.protocol_version = Some(theirs.to_owned()); + let label = format!("decoded {method:?} for {theirs}"); + assert_refused(engine, &label, &handle(request), INCOMPATIBLE_PROTOCOL_CODE); + } + } + + let unknown = format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","method":"frobnicate","payload":{{}}}}"# + ); + assert_refused( + engine, + &unknown, + &answer_text(&unknown), + UNKNOWN_METHOD_CODE, + ); + let no_method = format!(r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}"}}"#); + let by_position = format!(r#"["{ENGINE_PROTOCOL_VERSION}","capabilities",{{}}]"#); + for garbage in ["not json", no_method.as_str(), "[]", by_position.as_str()] { + assert_refused(engine, garbage, &answer_text(garbage), INVALID_REQUEST_CODE); + } + + let current = format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","method":"capabilities"}}"# + ); + let mut served = vec![current.as_str()]; + // A request with no version is one written before the field existed, + // `0.1.0`; it is served only while this build can serve that. + if super::engine_protocol_compatible(ENGINE_PROTOCOL_VERSION, None).is_ok() { + served.push(r#"{"method":"capabilities","payload":{}}"#); + } + for request in served { + let answer = answer_text(request); + assert!(answer.ok, "{engine}: {request}: {:?}", answer.error); + assert_eq!( + answer.protocol_version.as_deref(), + Some(ENGINE_PROTOCOL_VERSION), + "{engine}: {request}: a served answer must name this build's protocol" + ); + } + for method in [EngineMethod::Install, EngineMethod::Launch] { + let label = format!("decoded {method:?} with an empty payload"); + let answer = handle(EngineRequestEnvelope::new(method, serde_json::json!({}))); + assert_refused(engine, &label, &answer, INVALID_PAYLOAD_CODE); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EngineMethod; + use serde_json::json; + + /// The compatibility rule as a table, from this build's own version. + /// Runs on every platform, Windows included. + #[test] + fn the_caret_rule_from_this_builds_version() { + assert_eq!( + ENGINE_PROTOCOL_VERSION, "0.1.0", + "the table below is written for 0.1.0" + ); + let incompatible = |theirs: &str| ProtocolMismatch::Incompatible { + ours: "0.1.0".to_owned(), + theirs: theirs.to_owned(), + }; + let unreadable = |theirs: &str| ProtocolMismatch::Unreadable { + ours: "0.1.0".to_owned(), + theirs: theirs.to_owned(), + }; + let table: &[(Option<&str>, Result<(), ProtocolMismatch>)] = &[ + (None, Ok(())), + (Some("0.1.0"), Ok(())), + (Some("0.1.9"), Ok(())), + (Some("0.2.0"), Err(incompatible("0.2.0"))), + (Some("0.0.1"), Err(incompatible("0.0.1"))), + (Some("1.0.0"), Err(incompatible("1.0.0"))), + (Some("9.0.0"), Err(incompatible("9.0.0"))), + (Some(""), Err(unreadable(""))), + (Some("banana"), Err(unreadable("banana"))), + (Some("1.2"), Err(unreadable("1.2"))), + (Some("1.2.3.4"), Err(unreadable("1.2.3.4"))), + (Some("0.1.0-rc1"), Err(unreadable("0.1.0-rc1"))), + (Some("v0.1.0"), Err(unreadable("v0.1.0"))), + (Some("0..1"), Err(unreadable("0..1"))), + (Some("+0.1.0"), Err(unreadable("+0.1.0"))), + (Some("0.+1.0"), Err(unreadable("0.+1.0"))), + (Some("0.01.0"), Err(unreadable("0.01.0"))), + (Some("00.1.0"), Err(unreadable("00.1.0"))), + ( + Some("99999999999999999999.0.0"), + Err(unreadable("99999999999999999999.0.0")), + ), + ]; + for (theirs, expected) in table { + assert_eq!( + &engine_protocol_compatible(ENGINE_PROTOCOL_VERSION, *theirs), + expected, + "theirs: {theirs:?}" + ); + } + } + + /// From `1.0.0` on, only the major decides. + #[test] + fn past_one_the_major_alone_decides() { + assert_eq!(engine_protocol_compatible("1.2.0", Some("1.9.4")), Ok(())); + assert_eq!(engine_protocol_compatible("1.2.0", Some("1.0.0")), Ok(())); + assert!(engine_protocol_compatible("1.2.0", Some("2.0.0")).is_err()); + assert!(engine_protocol_compatible("1.2.0", Some("0.2.0")).is_err()); + // A peer that sends nothing is 0.1.0, which a 1.x side cannot use. + assert!(engine_protocol_compatible("1.0.0", None).is_err()); + } + + #[test] + fn this_builds_version_is_readable() { + assert!(parse_version(ENGINE_PROTOCOL_VERSION).is_some()); + assert!(parse_version(PRE_VERSIONING_PROTOCOL).is_some()); + } + + #[test] + fn a_request_this_engine_can_serve_gets_no_refusal() { + for version in [None, Some(ENGINE_PROTOCOL_VERSION), Some("0.1.7")] { + let mut request = EngineRequestEnvelope::new(EngineMethod::Stop, json!({})); + request.protocol_version = version.map(str::to_owned); + assert!( + incompatible_request_response(&request).is_none(), + "{version:?}" + ); + } + } + + #[test] + fn a_request_for_another_protocol_is_refused_naming_both_versions() { + let mut request = EngineRequestEnvelope::new(EngineMethod::Install, json!({})); + request.protocol_version = Some("0.2.0".to_owned()); + let response = incompatible_request_response(&request).expect("refused"); + assert!(!response.ok); + assert_eq!( + response.protocol_version.as_deref(), + Some(ENGINE_PROTOCOL_VERSION) + ); + let error = response.error.expect("an error"); + assert_eq!(error.code, INCOMPATIBLE_PROTOCOL_CODE); + assert_eq!( + error.message, + format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request was made for protocol 0.2.0, \ + which it cannot serve" + ) + ); + + request.protocol_version = Some("banana".to_owned()); + let error = incompatible_request_response(&request) + .expect("refused") + .error + .expect("an error"); + assert_eq!(error.code, INCOMPATIBLE_PROTOCOL_CODE); + assert_eq!( + error.message, + format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request named protocol version \ + \"banana\", which is not a MAJOR.MINOR.PATCH version, so it cannot be served" + ) + ); + } + + /// The handler a request reaches once its version and method are judged: + /// it serves `capabilities` and rejects every other payload, the shape of + /// a first-party engine for the cases the contract check sends. + fn toy_engine(request: EngineRequestEnvelope) -> EngineResponseEnvelope { + if let Some(refusal) = incompatible_request_response(&request) { + return refusal; + } + match request.method { + EngineMethod::Capabilities => EngineResponseEnvelope::success(json!({})), + _ => EngineResponseEnvelope::failure(INVALID_PAYLOAD_CODE, "toy"), + } + } + + /// A request is judged by its version before its method: one for another + /// protocol is refused even when its method or its version's type is one + /// this build cannot decode, and each other way a request can be unusable + /// still gets an answer that says so. + #[test] + fn a_request_is_judged_by_its_version_before_its_method() { + let answer = |text: &str| answer_engine_request(text, toy_engine); + let error = |text: &str| answer(text).error.expect("an error answer"); + + let refused = error(r#"{"protocol_version":"0.2.0","method":"frobnicate"}"#); + assert_eq!(refused.code, INCOMPATIBLE_PROTOCOL_CODE); + assert_eq!( + refused.message, + format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request was made for protocol 0.2.0, \ + which it cannot serve" + ) + ); + let not_a_string = error(r#"{"protocol_version":2,"method":"stop","payload":{}}"#); + assert_eq!(not_a_string.code, INCOMPATIBLE_PROTOCOL_CODE); + assert_eq!( + not_a_string.message, + format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request named protocol version \ + \"2\", which is not a MAJOR.MINOR.PATCH version, so it cannot be served" + ) + ); + + let unknown = error(&format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","method":"frobnicate"}}"# + )); + assert_eq!(unknown.code, UNKNOWN_METHOD_CODE); + assert_eq!( + unknown.message, + format!( + "this engine (engine protocol {ENGINE_PROTOCOL_VERSION}) has no method \"frobnicate\"" + ) + ); + assert_eq!( + error(r#"{"method":"frobnicate"}"#).code, + UNKNOWN_METHOD_CODE + ); + + let not_json = error("not json"); + assert_eq!(not_json.code, INVALID_REQUEST_CODE); + assert_eq!( + not_json.message, + "the request is not JSON: expected ident at line 1 column 2" + ); + let no_method = error(&format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}"}}"# + )); + assert_eq!(no_method.code, INVALID_REQUEST_CODE); + assert_eq!( + no_method.message, + "the request is not an engine request envelope: missing field `method`" + ); + + assert!(answer(r#"{"method":"capabilities"}"#).ok); + // A newer request on the same minor, with a field this build does not + // know, is served. + assert!(answer(r#"{"protocol_version":"0.1.9","method":"capabilities","hint":"new"}"#).ok); + assert!(answer(r#"{"protocol_version":null,"method":"capabilities"}"#).ok); + } + + /// The shared contract check passes an engine that holds the contract, + /// so a red result in an engine's own test is that engine's. + #[test] + fn the_contract_check_passes_an_engine_that_holds_it() { + engine_contract::assert_engine_holds_protocol_contract( + "toy", + |text| answer_engine_request(text, toy_engine), + toy_engine, + ); + } + + /// An engine that serves a request whatever protocol it names breaks the + /// contract, and the shared check says so: a check that passed any engine + /// would make every engine's contract test meaningless. + #[test] + #[should_panic(expected = "was served")] + fn the_contract_check_fails_an_engine_that_ignores_the_version() { + fn careless(request: EngineRequestEnvelope) -> EngineResponseEnvelope { + match request.method { + EngineMethod::Install | EngineMethod::Launch => { + EngineResponseEnvelope::failure(INVALID_PAYLOAD_CODE, "toy") + } + _ => EngineResponseEnvelope::success(json!({})), + } + } + engine_contract::assert_engine_holds_protocol_contract( + "careless", + |text| match serde_json::from_str::(text) { + Ok(request) => careless(request), + Err(_) => EngineResponseEnvelope::failure(INVALID_REQUEST_CODE, "toy"), + }, + careless, + ); + } + + /// An engine whose decoded path refuses but whose stdio path does not read + /// the version first fails the check too. + #[test] + #[should_panic(expected = "incompatible_protocol")] + fn the_contract_check_fails_an_engine_whose_stdio_skips_the_version() { + engine_contract::assert_engine_holds_protocol_contract( + "stdio-skips-version", + |text| match serde_json::from_str::(text) { + Ok(mut request) => { + request.protocol_version = None; + toy_engine(request) + } + Err(_) => EngineResponseEnvelope::failure(INVALID_REQUEST_CODE, "toy"), + }, + toy_engine, + ); + } + + /// The version the contract check uses as "incompatible" is the first one + /// the bump rule makes so, for both branches of the rule, and it is + /// incompatible with this build's. + #[test] + fn the_contract_checks_breaking_version_is_the_next_incompatible_one() { + assert_eq!(engine_contract::breaking_successor("0.1.0"), "0.2.0"); + assert_eq!(engine_contract::breaking_successor("0.3.7"), "0.4.0"); + assert_eq!(engine_contract::breaking_successor("1.4.2"), "2.0.0"); + assert_eq!(engine_contract::breaking_successor("2.0.0"), "3.0.0"); + let breaking = engine_contract::breaking_successor(ENGINE_PROTOCOL_VERSION); + assert!(engine_protocol_compatible(ENGINE_PROTOCOL_VERSION, Some(&breaking)).is_err()); + } + + /// The panic message of `check`, or `None` when it did not panic. + fn panic_message(check: impl FnOnce() + std::panic::UnwindSafe) -> Option { + let payload = std::panic::catch_unwind(check).err()?; + Some( + payload + .downcast_ref::() + .cloned() + .or_else(|| { + payload + .downcast_ref::<&str>() + .map(|text| (*text).to_owned()) + }) + .unwrap_or_default(), + ) + } + + /// Six rules of the shared contract check, each failing an engine that + /// breaks that rule alone: a refusal names this build's protocol; a served + /// answer names it; an unknown method in another protocol is refused for + /// the protocol on the stdio path; the decoded path refuses another + /// protocol; a request with no version is served; an empty-payload install + /// reaches the engine's own decoding. Every toy below is the conforming toy + /// with one of these wrong, and the check must panic naming it. The check's + /// other rules have no toy here. + #[test] + fn the_contract_check_fails_an_engine_that_breaks_any_one_rule() { + type Answer = fn(&str) -> EngineResponseEnvelope; + type Handle = fn(EngineRequestEnvelope) -> EngineResponseEnvelope; + fn conforming_text(text: &str) -> EngineResponseEnvelope { + answer_engine_request(text, toy_engine) + } + fn unversioned(mut answer: EngineResponseEnvelope) -> EngineResponseEnvelope { + answer.protocol_version = None; + answer + } + // Rule: a refusal names this build's protocol. + fn refuses_unversioned(text: &str) -> EngineResponseEnvelope { + let answer = conforming_text(text); + if answer.ok { + answer + } else { + unversioned(answer) + } + } + // Rule: a served answer names this build's protocol. + fn serves_unversioned(text: &str) -> EngineResponseEnvelope { + let answer = conforming_text(text); + if answer.ok { + unversioned(answer) + } else { + answer + } + } + // Rule: an unknown method in another protocol is refused for the + // protocol, not for the method. + fn method_before_version(text: &str) -> EngineResponseEnvelope { + if text.contains(r#""method":"frobnicate""#) { + return EngineResponseEnvelope::failure(UNKNOWN_METHOD_CODE, "toy"); + } + conforming_text(text) + } + // Rule: the decoded path refuses another protocol too. + fn decoded_ignores_version(request: EngineRequestEnvelope) -> EngineResponseEnvelope { + let mut request = request; + request.protocol_version = None; + toy_engine(request) + } + // Rule: a request with no version is served as `0.1.0`. + fn refuses_unversioned_requests(text: &str) -> EngineResponseEnvelope { + let value: serde_json::Value = serde_json::from_str(text).unwrap_or_default(); + if value.get("method").is_some() && value.get("protocol_version").is_none() { + return refusal_for(Some("banana")).expect("refused"); + } + conforming_text(text) + } + // Rule: an install or launch with an empty payload reaches the + // engine's own decoding. + fn installs_anything(request: EngineRequestEnvelope) -> EngineResponseEnvelope { + if let Some(refusal) = incompatible_request_response(&request) { + return refusal; + } + EngineResponseEnvelope::success(json!({})) + } + let breaking = engine_contract::breaking_successor(ENGINE_PROTOCOL_VERSION); + let cases: [(&str, Answer, Handle, String); 6] = [ + ( + "refusal-version", + refuses_unversioned, + toy_engine, + "the answer must name this build's protocol".to_owned(), + ), + ( + "served-version", + serves_unversioned, + toy_engine, + "a served answer must name this build's protocol".to_owned(), + ), + ( + "method-first", + method_before_version, + toy_engine, + format!(r#"{{"protocol_version":"{breaking}","method":"frobnicate""#), + ), + ( + "decoded-version", + conforming_text, + decoded_ignores_version, + format!("decoded Detect for {breaking}: "), + ), + ( + "legacy-request", + refuses_unversioned_requests, + toy_engine, + r#"{"method":"capabilities","payload":{}}"#.to_owned(), + ), + ( + "empty-install", + conforming_text, + installs_anything, + "decoded Install with an empty payload was served".to_owned(), + ), + ]; + for (label, answer_text, handle, expected) in cases { + let message = panic_message(|| { + engine_contract::assert_engine_holds_protocol_contract(label, answer_text, handle); + }) + .unwrap_or_else(|| panic!("{label}: the check passed an engine that breaks it")); + assert!(message.contains(&expected), "{label}: {message}"); + } + } +} diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 2dd776d15b..7b30df2d9a 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -4,7 +4,9 @@ //! The client side of the engine stdio protocol: run an engine as a child //! process, send it one [`EngineRequestEnvelope`] on stdin as a single line of -//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. +//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. Each child +//! serves one request; an `install` or a `launch` starts two, one after the +//! other: the `capabilities` check described below, then the request itself. //! //! `rocm` and `rocmd` both talk to engines this way and share this one //! implementation. They differ in *which* process they start (`rocm` runs the @@ -16,8 +18,29 @@ //! unusable answer is reported as, and decoding the response. The few things //! the callers want differently are the fields of [`EngineStdioOptions`]. //! +//! It is also where the engine protocol version is enforced on the client side +//! (the rule is in the `protocol_version` module). An answer's +//! `protocol_version` is checked before anything else in it: by +//! [`decode_engine_response`] for an answer that decodes as an envelope, and +//! from the raw JSON object for one that does not, before its shape is. A +//! stdout that is not a single JSON object has no version to check and is +//! reported as unreadable. [`run_engine_stdio`], before an `install` or a +//! `launch`, first sends a +//! `capabilities` request through the same path and does not send the real one +//! unless that answer is in a protocol this build can use. The other methods are +//! sent once, as they are. +//! +//! The whole of an engine's stdout must be its one answer: an engine writes +//! only its JSON answer to stdout and its logs to stderr. Output that is not +//! exactly one envelope is refused, never salvaged, and the error says what was +//! wrong with it: nothing printed; other output before or after an answer, +//! which it quotes; or JSON that is not an envelope, or not JSON, quoting the +//! start of stdout. +//! //! The request and the child's lifetime are not bounded by a timeout: an engine -//! install legitimately runs for minutes, and no caller has ever set one. +//! install legitimately runs for minutes, and no caller has ever set one. That +//! holds for the `capabilities` child that goes before an `install` or a +//! `launch` too, so those requests wait, unbounded, on two children in turn. //! //! Nor is the wait for the pipes after the child has exited, and that is a known //! limit rather than an oversight. The answer is read to the end of the child's @@ -25,23 +48,26 @@ //! grandchild holding either pipe open makes the request wait for that grandchild //! to exit. Both clients this one replaces behaved the same way (`rocm` read //! stdout to the end then joined its stderr thread; `rocmd` used -//! `wait_with_output`), so nothing new is exposed to it, but the daemon is long -//! lived and a hung request holds its caller. Bounding it exactly would mean -//! reading both pipes on their own threads, waiting for the child itself without a -//! limit and for the pipes only up to a deadline after it exits, and choosing what -//! to report when the deadline passes (an answer already complete? a partial -//! one?). That is a timeout policy for an engine contract nothing here sets, so -//! this change leaves it to be decided on its own. The one wait this client adds -//! itself, for the stderr of a child whose request could not be written, is -//! bounded by `EXITED_STDERR_GRACE`. +//! `wait_with_output`). The `capabilities` check before an `install` or a +//! `launch` is a second child exposed to the same wait, so an engine that +//! leaves a grandchild holding a pipe can hang the check as well as the +//! request; the daemon is long lived and a hung request holds its caller. +//! Bounding it exactly would mean reading both pipes on their own threads, +//! waiting for the child itself without a limit and for the pipes only up to a +//! deadline after it exits, and choosing what to report when the deadline +//! passes (an answer already complete? a partial one?). That is a timeout +//! policy for an engine contract nothing here sets, so it is left to be decided +//! on its own. The one wait this client adds itself, for the stderr of a child +//! whose request could not be written, is bounded by `EXITED_STDERR_GRACE`. //! //! A request that cannot be written is the other place the answer-wins rule does //! not apply: if the engine answered and exited without draining a request larger //! than a pipe buffer, the write fails, the child is killed and its stdout is //! never read, so the error carries its stderr and not its answer. Every engine //! here (`lemonade`, `vllm`, `aim`, and the `rocm __engine-stdio` verb that serves -//! the built-in ones) reads stdin to the end before it answers, so reaching this -//! needs an engine that breaks the protocol. Decoding such an answer would mean +//! the built-in ones, once it has accepted the engine name) reads stdin to the end +//! before it answers, so reaching this needs an engine that breaks the +//! protocol. Decoding such an answer would mean //! waiting, up to a deadline, for the child to finish printing it and to exit //! before deciding it is complete, and an engine that has stopped reading is no //! engine to wait for, so it is left as a documented limit. @@ -69,7 +95,7 @@ //! thread, so an engine that fills it before draining a large request cannot //! deadlock the write. -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::io::{self, Read, Write}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; @@ -84,7 +110,8 @@ use serde::Serialize; use serde::de::DeserializeOwned; use crate::{ - ENDPOINT_API_KEY_FILE_ENV, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, + ENDPOINT_API_KEY_FILE_ENV, ENGINE_PROTOCOL_VERSION, EngineMethod, EngineRequestEnvelope, + EngineResponseEnvelope, ProtocolMismatch, engine_protocol_compatible, }; /// Set to `1` on an engine child whose caller wants its progress lines on @@ -124,8 +151,8 @@ pub struct EngineStdioOptions<'a> { /// shown in the error when the request fails. pub forward_stderr: bool, /// Say, on this process's stderr, when the child answered and then exited - /// with a failing status. The answer is used either way; this only leaves a - /// trace of the status, which the caller would otherwise never see. For a + /// with a failing status. The answer is read either way (and then judged + /// like any other); this only leaves a trace of the status, which the caller would otherwise never see. For a /// daemon, whose stderr is its log; a terminal user has no use for it. pub note_failing_exit_after_answer: bool, } @@ -135,11 +162,10 @@ pub fn engine_request_envelope( method: EngineMethod, request: &T, ) -> Result { - Ok(EngineRequestEnvelope { + Ok(EngineRequestEnvelope::new( method, - payload: serde_json::to_value(request) - .context("failed to encode engine request payload")?, - }) + serde_json::to_value(request).context("failed to encode engine request payload")?, + )) } /// The command [`run_engine_stdio`] runs: `program` with `args`, all three @@ -187,9 +213,20 @@ where /// non-zero is read as that envelope, which says more than the status does. The /// same holds for a success envelope (`ok: true`): a child that answers /// successfully and then exits non-zero is reported as the success it answered, -/// and the failing exit is not reported or logged anywhere, by design, because -/// the answer is what the caller asked for. -/// Every other failure names what went wrong and carries the child's stderr. +/// and the failing exit is not an error, because the answer is what the caller +/// asked for; it is only noted on this process's stderr when the caller sets +/// `note_failing_exit_after_answer`. +/// Every other failure names what went wrong and carries the child's stderr, +/// with one exception: when the capabilities check below refuses the request +/// on that child's answer (a protocol mismatch or an error answer), the error +/// carries neither the check child's stderr nor, for a mismatch, its exit +/// status, as [`check_protocol_before`] documents. +/// +/// For a method that installs or starts something +/// ([`EngineMethod::requires_protocol_handshake`]), the engine is first asked +/// for its capabilities, as a request of its own, and `envelope` is sent only +/// when that answer is in a compatible protocol and is not an error; otherwise +/// the error says `envelope` was not sent, and why. pub fn run_engine_stdio( program: &Path, args: I, @@ -224,6 +261,110 @@ where I: IntoIterator, S: AsRef, { + let args: Vec = args + .into_iter() + .map(|arg| arg.as_ref().to_owned()) + .collect(); + if envelope.method.requires_protocol_handshake() { + check_protocol_before(program, &args, options, &envelope.method)?; + } + exchange(program, &args, options, envelope, forward) +} + +/// Before a request that installs or starts something (see +/// [`EngineMethod::requires_protocol_handshake`]), ask the engine for its +/// capabilities, through the same path, and check the protocol it answers in. +/// The request itself is sent only when that answer is in a protocol this +/// build can use; otherwise the error says it was not sent, and why. +/// +/// Nothing is cached: an engine can be replaced between two requests, and the +/// extra child is small next to the install or launch it guards. The +/// capabilities child runs with [`handshake_options`]; its stderr is not +/// forwarded (it is in the error when asking for its capabilities fails, and +/// nowhere when the check fails on its answer), and it sends an empty +/// object as its payload, so an engine that decodes every payload as an object +/// still answers it. +fn check_protocol_before( + program: &Path, + args: &[OsString], + options: &EngineStdioOptions<'_>, + method: &EngineMethod, +) -> Result<()> { + let refused = format!("the {} request was not sent to the engine", method.name()); + let capabilities = + EngineRequestEnvelope::new(EngineMethod::Capabilities, serde_json::json!({})); + let answer = exchange( + program, + args, + &handshake_options(options), + &capabilities, + None::, + ) + .map_err(|error| match error.downcast::() { + Ok(AnswerInOtherProtocol { mismatch, .. }) => handshake_mismatch(&refused, mismatch), + Err(error) => anyhow::anyhow!( + "{refused}: its protocol could not be checked against this release's \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) because asking for its capabilities \ + failed: {error:#}" + ), + })?; + if let Err(mismatch) = + engine_protocol_compatible(ENGINE_PROTOCOL_VERSION, answer.protocol_version.as_deref()) + { + return Err(handshake_mismatch(&refused, mismatch)); + } + if !answer.ok { + bail!( + "{refused}: its protocol could not be checked against this release's \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) because it refused the capabilities \ + request: {}", + engine_error_text(answer.error) + ); + } + Ok(()) +} + +/// The error for a capabilities answer in a protocol this build cannot use: +/// the request named by `refused` was not sent, and why. The same words whether +/// the answer decoded as an envelope or was judged from its raw object. +fn handshake_mismatch(refused: &str, mismatch: ProtocolMismatch) -> anyhow::Error { + match mismatch { + ProtocolMismatch::Incompatible { ours, theirs } => anyhow::anyhow!( + "{refused}: the engine speaks engine protocol {theirs}, which this release \ + (engine protocol {ours}) cannot use" + ), + ProtocolMismatch::Unreadable { ours, theirs } => anyhow::anyhow!( + "{refused}: the engine named its protocol version {theirs:?}, which is not a \ + MAJOR.MINOR.PATCH version, so this release (engine protocol {ours}) cannot tell \ + whether it can use it" + ), + } +} + +/// The options the capabilities child of [`check_protocol_before`] runs with: +/// the real request's environment (app paths, engine environments root, the +/// endpoint key file and whether an inherited one is withheld), so the engine +/// that is checked is the one the request will reach, without the request for +/// progress, which is the install's, and without the note on a failing exit, +/// since the check reports its own outcome. +const fn handshake_options<'a>(options: &EngineStdioOptions<'a>) -> EngineStdioOptions<'a> { + EngineStdioOptions { + request_progress: false, + note_failing_exit_after_answer: false, + ..*options + } +} + +/// One request and its answer: start the child, write `envelope`, read what it +/// printed. [`run_engine_stdio_forwarding`] is this plus the protocol check that +/// goes before a request that installs or starts something. +fn exchange( + program: &Path, + args: &[OsString], + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, + forward: Option, +) -> Result { let mut child = engine_stdio_command(program, args, options) .spawn() .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; @@ -284,9 +425,17 @@ where } bail!("{exited}: {stderr}"); } - let answer = serde_json::from_slice(&stdout) - .with_context(|| unparseable_answer_message(status, &stderr)); - // The note says the answer was used, so it is only made for an answer that + let answer = parse_answer(&stdout).ok_or_else(|| { + match incompatible_object(ENGINE_PROTOCOL_VERSION, &stdout) { + Some(mismatch) => anyhow::Error::new(AnswerInOtherProtocol { + mismatch, + exited: (!status.success()).then(|| exit_note(status)), + stderr: stderr.clone(), + }), + None => anyhow::anyhow!(unparseable_answer_message(status, &stdout, &stderr)), + } + }); + // The note says an answer was read, so it is only made for an answer that // parsed; an unparseable one is reported by the error above, with the status. if answer.is_ok() && !status.success() && options.note_failing_exit_after_answer { note_failing_exit_after_answer(&mut io::stderr(), status); @@ -300,7 +449,7 @@ where /// in `rocm` (SIGPIPE at its default action) nor panic it, as `eprintln!` would. fn note_failing_exit_after_answer(sink: &mut impl Write, status: ExitStatus) { let note = format!( - "{} after printing its answer; the answer was used\n", + "{} after printing its answer; the answer was read\n", exit_note(status) ); forward_best_effort(sink, note.as_bytes()); @@ -326,18 +475,237 @@ fn exit_note(status: ExitStatus) -> String { /// The error for an answer that is not a response envelope. A child that also /// exited with a failing status says so, because that is often why the answer is -/// cut short; the stderr it wrote follows when there is any. -fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { +/// cut short; then what was wrong with what it printed ([`unreadable_stdout`]); +/// the stderr it wrote follows when there is any. +fn unparseable_answer_message(status: ExitStatus, stdout: &[u8], stderr: &str) -> String { let mut message = "failed to parse engine response envelope".to_owned(); if !status.success() { message = format!("{message} ({})", exit_note(status)); } + message = format!("{message}: {}", unreadable_stdout(stdout)); if !stderr.is_empty() { message = format!("{message}; stderr: {stderr}"); } message } +/// The answer envelope `stdout` holds, when the whole of it is one: a single +/// JSON object that decodes as an envelope. +fn parse_answer(stdout: &[u8]) -> Option { + answer_from_value(serde_json::from_slice(stdout).ok()?) +} + +/// An answer that does not decode as an envelope and names a protocol this +/// build cannot use. Its message is the one for any answer in such a protocol, +/// followed by the child's failing exit status and its stderr when there are +/// any, as for every other unusable answer; the type lets the protocol check +/// before an install or launch report the mismatch alone. +#[derive(Debug)] +struct AnswerInOtherProtocol { + mismatch: ProtocolMismatch, + /// The failing exit, when the child did not exit cleanly. + exited: Option, + /// What the child wrote to stderr, trimmed; empty for none. + stderr: String, +} + +impl std::fmt::Display for AnswerInOtherProtocol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&incompatible_answer_message(&self.mismatch))?; + if let Some(exited) = &self.exited { + write!(f, " ({exited})")?; + } + if !self.stderr.is_empty() { + write!(f, "; stderr: {}", self.stderr)?; + } + Ok(()) + } +} + +impl std::error::Error for AnswerInOtherProtocol {} + +/// The protocol mismatch of a stdout that is one JSON object naming a protocol +/// this build cannot use, or `None`. Used for an object that does not decode +/// as an envelope, so its version is judged from the raw object before the +/// shape of the rest is: an answer in another protocol may have another shape, +/// and the mismatch is what has to be fixed. +fn incompatible_object(ours: &str, stdout: &[u8]) -> Option { + let value: serde_json::Value = serde_json::from_slice(stdout).ok()?; + if !value.is_object() { + return None; + } + let theirs = crate::protocol_version::raw_protocol_version(&value); + engine_protocol_compatible(ours, theirs.as_deref()).err() +} + +/// `value` as an answer envelope, when it is a JSON object that decodes as one. +/// A derived struct also decodes from a JSON array of its fields in order +/// (`[null,true,null,null]`), which no engine writes, so anything but an +/// object is refused rather than read by position. +fn answer_from_value(value: serde_json::Value) -> Option { + if !value.is_object() { + return None; + } + serde_json::from_value(value).ok() +} + +/// What an engine's stdout must hold, said wherever it held something else. +const STDOUT_RULE: &str = "an engine writes only its JSON answer to stdout; logs go to stderr"; + +/// How much of an engine's output an error quotes: the first non-blank line of +/// an unreadable stdout, or a decoder's message about it (which can quote a +/// value the engine wrote in full), cut to this many characters. +const STDOUT_EXCERPT_CHARS: usize = 120; + +/// How many pieces of stdout (a line of text, or one whole top-level JSON +/// value however many lines it spans) [`answer_among_other_output`] reads, so a +/// large unreadable stdout costs a bounded amount of work to describe. +const STDOUT_SEARCH_PIECES: usize = 256; + +/// Where the other output is, in a stdout that holds an answer and something +/// else, with an excerpt of that other output. +#[derive(Debug, PartialEq, Eq)] +enum OtherOutput { + BeforeAnswer(String), + AfterAnswer(String), +} + +/// What is wrong with a stdout that does not parse as one response envelope, +/// for the error that reports it. +/// +/// The whole of stdout must be the answer, so nothing here salvages one: an +/// answer found among other output is still refused, and only named as the +/// reason. The cases, in the order they are told apart: +/// - nothing (or only whitespace) was printed; +/// - a top-level envelope is there, but so is other output before or after it +/// (a log line on the wrong stream), which is what the error quotes; +/// - the whole of it is one JSON value, but not an envelope; +/// - it is not JSON. +fn unreadable_stdout(stdout: &[u8]) -> String { + if stdout.iter().all(u8::is_ascii_whitespace) { + return "the engine printed no answer on stdout".to_owned(); + } + match answer_among_other_output(stdout) { + Some(OtherOutput::BeforeAnswer(other)) => { + return format!( + "the engine wrote other output to stdout before its answer, starting with \ + `{other}` ({STDOUT_RULE})" + ); + } + Some(OtherOutput::AfterAnswer(other)) => { + return format!( + "the engine wrote other output to stdout after its answer, starting with \ + `{other}` ({STDOUT_RULE})" + ); + } + None => {} + } + let excerpt = stdout_excerpt(stdout); + match serde_json::from_slice::(stdout) { + Ok(value) => { + let reason = if value.is_object() { + serde_json::from_slice::(stdout) + .err() + .map_or_else(String::new, |error| { + format!(" ({})", bounded(&error.to_string())) + }) + } else { + " (an answer is a JSON object)".to_owned() + }; + format!( + "the engine's stdout is JSON but not an answer envelope{reason}, starting with \ + `{excerpt}` ({STDOUT_RULE})" + ) + } + Err(error) => format!( + "the engine's stdout is not JSON ({}), starting with `{excerpt}` \ + ({STDOUT_RULE})", + bounded(&error.to_string()) + ), + } +} + +/// The other output beside a response envelope in `stdout`, when the envelope +/// is a top-level value there and something else is too, or `None`. +/// +/// `stdout` is read as a sequence of pieces: wherever (after whitespace) a `{` +/// or `[` starts a whole JSON value, that value, however many lines it spans, +/// since engines pretty-print their answer; anything else, a line of text. Only +/// a top-level object can be the answer, so an envelope nested inside another +/// value is never mistaken for one. At most [`STDOUT_SEARCH_PIECES`] pieces +/// are read. +fn answer_among_other_output(stdout: &[u8]) -> Option { + let mut position = 0; + let mut answered = false; + let mut first_other: Option = None; + for _ in 0..STDOUT_SEARCH_PIECES { + position += stdout[position..] + .iter() + .take_while(|byte| byte.is_ascii_whitespace()) + .count(); + let rest = &stdout[position..]; + if rest.is_empty() { + return None; + } + if answered { + return Some(OtherOutput::AfterAnswer(stdout_excerpt(rest))); + } + if matches!(rest[0], b'{' | b'[') { + let mut values = + serde_json::Deserializer::from_slice(rest).into_iter::(); + if let Some(Ok(value)) = values.next() { + let is_answer = answer_from_value(value).is_some(); + if is_answer { + if let Some(start) = first_other { + return Some(OtherOutput::BeforeAnswer(stdout_excerpt(&stdout[start..]))); + } + answered = true; + } else { + first_other.get_or_insert(position); + } + position += values.byte_offset(); + continue; + } + } + first_other.get_or_insert(position); + match rest.iter().position(|byte| *byte == b'\n') { + Some(newline) => position += newline + 1, + None => return None, + } + } + None +} + +/// The first non-blank line of `stdout`, [`bounded`]. +fn stdout_excerpt(stdout: &[u8]) -> String { + let text = String::from_utf8_lossy(stdout); + let line = text + .lines() + .map(str::trim) + .find(|line| !line.is_empty()) + .unwrap_or_default(); + bounded(line) +} + +/// `text` cut to [`STDOUT_EXCERPT_CHARS`] characters, with `...` marking a +/// cut, and with control characters (`char::is_control`) escaped. Other +/// characters, such as Unicode format characters, are quoted as they are. +fn bounded(text: &str) -> String { + let mut excerpt = String::new(); + for (index, ch) in text.chars().enumerate() { + if index == STDOUT_EXCERPT_CHARS { + excerpt.push_str("..."); + break; + } + if ch.is_control() { + excerpt.extend(ch.escape_default()); + } else { + excerpt.push(ch); + } + } + excerpt +} + /// `pipe` of `child`, or an error that also kills and reaps the child: a child /// whose request cannot be sent must not be left running, as on every other /// failure path. Unreachable while the command pipes all three streams, which is @@ -636,21 +1004,55 @@ fn forward_best_effort(sink: &mut impl Write, bytes: &[u8]) { /// Turn an engine's answer into the response type the caller asked for: the /// engine's own error when it reported one, otherwise the decoded `data`. +/// +/// The protocol the answer was written in is checked first, before `ok`: an +/// answer in a protocol this build cannot use is refused whatever it says, +/// and so is an engine's own error in such a protocol, because its meaning +/// cannot be trusted either and the mismatch is the thing to fix. pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result where R: DeserializeOwned, { + if let Err(mismatch) = engine_protocol_compatible( + ENGINE_PROTOCOL_VERSION, + envelope.protocol_version.as_deref(), + ) { + bail!("{}", incompatible_answer_message(&mismatch)); + } if !envelope.ok { - let error = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |value| format!("{}: {}", value.code, value.message), - ); - bail!("{error}"); + bail!("{}", engine_error_text(envelope.error)); } let data = envelope .data .context("engine response envelope did not contain data")?; - serde_json::from_value(data).context("failed to decode engine response payload") + serde_json::from_value(data).map_err(|error| { + anyhow::anyhow!( + "failed to decode engine response payload: {}", + bounded(&error.to_string()) + ) + }) +} + +/// The error for an answer in a protocol this build cannot use. +fn incompatible_answer_message(mismatch: &ProtocolMismatch) -> String { + match mismatch { + ProtocolMismatch::Incompatible { ours, theirs } => format!( + "engine answered in engine protocol {theirs}, which this release \ + (engine protocol {ours}) cannot use" + ), + ProtocolMismatch::Unreadable { ours, theirs } => format!( + "engine answered with protocol version {theirs:?}, which is not a \ + MAJOR.MINOR.PATCH version this release (engine protocol {ours}) can read" + ), + } +} + +/// An engine's own error, as `code: message`. +fn engine_error_text(error: Option) -> String { + error.map_or_else( + || "unknown engine error".to_owned(), + |value| format!("{}: {}", value.code, value.message), + ) } #[cfg(test)] @@ -697,7 +1099,7 @@ mod tests { /// /// `rocm` passes `endpoint_key_file: None` for every external engine /// plugin — a third-party binary — and the path it would otherwise carry - /// points at a mode-0600 file holding a live endpoint credential. Merely + /// points at an owner-only file holding a live endpoint credential. Merely /// not setting it would still hand the value through whenever this process /// inherited one. /// @@ -839,13 +1241,16 @@ mod tests { assert_eq!(envelope.payload, json!({"service_id": "svc"})); assert_eq!( serde_json::to_string(&envelope).unwrap(), - r#"{"method":"stop","payload":{"service_id":"svc"}}"# + format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","method":"stop","payload":{{"service_id":"svc"}}}}"# + ) ); } #[test] fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { let failed = EngineResponseEnvelope { + protocol_version: None, ok: false, data: None, error: Some(EngineErrorDetail { @@ -857,6 +1262,7 @@ mod tests { assert_eq!(error.to_string(), "stop_failed: it broke"); let unexplained = EngineResponseEnvelope { + protocol_version: None, ok: false, data: None, error: None, @@ -865,6 +1271,7 @@ mod tests { assert_eq!(error.to_string(), "unknown engine error"); let empty = EngineResponseEnvelope { + protocol_version: None, ok: true, data: None, error: None, @@ -879,7 +1286,341 @@ mod tests { let error = decode_engine_response::(wrong_shape).unwrap_err(); assert_eq!( error.to_string(), - "failed to decode engine response payload" + "failed to decode engine response payload: \ + invalid type: string \"yes\", expected a boolean" + ); + } + + /// An answer, as a client reads it, from an engine that wrote it in + /// `version` (`None`: before the field existed). + fn answer_in(version: Option<&str>, ok: bool) -> EngineResponseEnvelope { + let mut answer = if ok { + EngineResponseEnvelope::success(json!({"stopped": true, "graceful": true})) + } else { + EngineResponseEnvelope::failure("incompatible_protocol", "the engine's own refusal") + }; + answer.protocol_version = version.map(str::to_owned); + answer + } + + /// The answer's protocol is checked before anything else in it: a success + /// with perfectly good data is refused when it is in a protocol this + /// release cannot use, and an engine's own error in such a protocol is + /// reported as the mismatch, which is what has to be fixed. + #[test] + fn an_answer_in_an_incompatible_protocol_is_refused_whatever_it_says() { + for ok in [true, false] { + let error = + decode_engine_response::(answer_in(Some("9.0.0"), ok)).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "engine answered in engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + "ok: {ok}" + ); + let error = + decode_engine_response::(answer_in(Some("0.2.0"), ok)).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "engine answered in engine protocol 0.2.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + "ok: {ok}" + ); + let error = + decode_engine_response::(answer_in(Some("banana"), ok)).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "engine answered with protocol version \"banana\", which is not a \ + MAJOR.MINOR.PATCH version this release (engine protocol \ + {ENGINE_PROTOCOL_VERSION}) can read" + ), + "ok: {ok}" + ); + } + } + + /// The same answer in this protocol, in a newer one with only additions, + /// or from an engine older than the field, is read as answered. + #[test] + fn an_answer_in_a_compatible_or_unnamed_protocol_is_read() { + for version in [Some(ENGINE_PROTOCOL_VERSION), Some("0.1.9"), None] { + let decoded = decode_engine_response::(answer_in(version, true)) + .unwrap_or_else(|error| panic!("{version:?}: {error:#}")); + assert!(decoded.stopped, "{version:?}"); + let error = + decode_engine_response::(answer_in(version, false)).unwrap_err(); + assert_eq!( + error.to_string(), + "incompatible_protocol: the engine's own refusal", + "{version:?}" + ); + } + } + + /// An object that is not a valid envelope is judged by the protocol it + /// names first: in another protocol, or with a version that is not text, + /// the error is the mismatch and not the stdout rule; in this protocol (or + /// none), the shape of the object is what is reported. + #[test] + fn a_malformed_answer_is_judged_by_its_protocol_before_its_shape() { + let incompatible = format!( + "engine answered in engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ); + for stdout in [ + r#"{"protocol_version":"9.0.0","ok":"yes"}"#, + r#"{"protocol_version":"9.0.0","status":"ok"}"#, + r#"{"protocol_version":"9.0.0","ok":false,"error":"boom"}"#, + ] { + assert_eq!( + incompatible_object(ENGINE_PROTOCOL_VERSION, stdout.as_bytes()) + .map(|mismatch| incompatible_answer_message(&mismatch)) + .as_deref(), + Some(incompatible.as_str()), + "{stdout}" + ); + } + assert_eq!( + incompatible_object( + ENGINE_PROTOCOL_VERSION, + br#"{"protocol_version":9,"ok":true,"data":{}}"# + ) + .map(|mismatch| incompatible_answer_message(&mismatch)) + .as_deref(), + Some( + format!( + "engine answered with protocol version \"9\", which is not a \ + MAJOR.MINOR.PATCH version this release (engine protocol \ + {ENGINE_PROTOCOL_VERSION}) can read" + ) + .as_str() + ) + ); + // An array is not an object, so it is not judged at all: read as an + // absent version it would be `0.1.0`, which a `0.2.0` build cannot use. + assert_eq!(incompatible_object("0.2.0", b"[null,true,null,null]"), None); + assert!(incompatible_object("0.2.0", br#"{"ok":true}"#).is_some()); + let current = format!(r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":"yes"}}"#); + for compatible in [current.as_str(), r#"{"ok":"yes"}"#] { + assert_eq!( + incompatible_object(ENGINE_PROTOCOL_VERSION, compatible.as_bytes()), + None + ); + assert!( + unreadable_stdout(compatible.as_bytes()) + .starts_with("the engine's stdout is JSON but not an answer envelope"), + "{compatible}" + ); + } + } + + /// Each way a stdout can fail to be one envelope is told apart, and none of + /// them is salvaged. + #[test] + fn an_unreadable_stdout_is_described_by_what_is_wrong_with_it() { + const RULE: &str = "(an engine writes only its JSON answer to stdout; logs go to stderr)"; + let envelope = format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":true,"data":{{"stopped":true}}}}"# + ); + let cases: Vec<(String, String)> = vec![ + ( + String::new(), + "the engine printed no answer on stdout".to_owned(), + ), + ( + " \n\t\n".to_owned(), + "the engine printed no answer on stdout".to_owned(), + ), + ( + format!("INFO loading model\n{envelope}\n"), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `INFO loading model` {RULE}" + ), + ), + ( + // Several pieces before the answer: the first is quoted. + format!("first\nsecond\n{envelope}\n"), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `first` {RULE}" + ), + ), + ( + // A line of text, then a JSON value that is not an answer, then + // the answer: the line comes first and is quoted. + format!("loading\n{{\"progress\":1}}\n{envelope}\n"), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `loading` {RULE}" + ), + ), + ( + // A JSON value that is not an answer, then a line of text, then + // the answer: the JSON value comes first and is quoted. + format!("{{\"progress\":1}}\nloading\n{envelope}\n"), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `{{\"progress\":1}}` {RULE}" + ), + ), + ( + format!("{envelope}\ndone\n"), + format!( + "the engine wrote other output to stdout after its answer, starting \ + with `done` {RULE}" + ), + ), + ( + format!("{envelope}{envelope}"), + format!( + "the engine wrote other output to stdout after its answer, starting \ + with `{envelope}` {RULE}" + ), + ), + ( + "[\n {\"ok\": true}\n]\n".to_owned(), + format!( + "the engine's stdout is JSON but not an answer envelope (an answer is a \ + JSON object), starting with `[` {RULE}" + ), + ), + ( + // The fields of an envelope in order, which a derived struct + // would decode by position: not an answer, however it reads. + "[null,true,null,null]".to_owned(), + format!( + "the engine's stdout is JSON but not an answer envelope (an answer is a \ + JSON object), starting with `[null,true,null,null]` {RULE}" + ), + ), + ( + format!("[null,true,null,null]\n{envelope}\n"), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `[null,true,null,null]` {RULE}" + ), + ), + ( + "progress\n{\"result\":\n{\"ok\": true}\n}\n".to_owned(), + format!( + "the engine's stdout is not JSON (expected value at line 1 column 1), \ + starting with `progress` {RULE}" + ), + ), + ( + "loading {model}\n {\n \"ok\": true,\n \"data\": {}\n }\n".to_owned(), + format!( + "the engine wrote other output to stdout before its answer, starting \ + with `loading {{model}}` {RULE}" + ), + ), + ( + r#"{"status":"ok"}"#.to_owned(), + format!( + "the engine's stdout is JSON but not an answer envelope (missing field `ok` \ + at line 1 column 15), starting with `{{\"status\":\"ok\"}}` {RULE}" + ), + ), + ( + "not json".to_owned(), + format!( + "the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` {RULE}" + ), + ), + ( + "\u{1b}[31mred\u{7}\n".to_owned(), + format!( + "the engine's stdout is not JSON (expected value at line 1 column 1), \ + starting with `\\u{{1b}}[31mred\\u{{7}}` {RULE}" + ), + ), + ]; + for (stdout, expected) in cases { + assert_eq!(unreadable_stdout(stdout.as_bytes()), expected, "{stdout:?}"); + } + + // A decoder's message can quote what the engine wrote in full; the + // error quotes it bounded, like the excerpt beside it. + let long = format!(r#"{{"ok":false,"error":"{}"}}"#, "x".repeat(10_000)); + let message = unreadable_stdout(long.as_bytes()); + assert!(message.len() < 600, "{} bytes: {message}", message.len()); + assert!( + message.starts_with( + "the engine's stdout is JSON but not an answer envelope (invalid type: string \"x" + ), + "{message}" + ); + let excerpt: String = long.chars().take(STDOUT_EXCERPT_CHARS).collect(); + assert!(message.contains(&format!("`{excerpt}...`")), "{message}"); + let payload = EngineResponseEnvelope::success(json!({"stopped": "x".repeat(10_000)})); + let error = decode_engine_response::(payload) + .unwrap_err() + .to_string(); + assert!(error.len() < 300, "{} bytes: {error}", error.len()); + assert!( + error.starts_with("failed to decode engine response payload: invalid type: string \"x"), + "{error}" + ); + assert!(error.ends_with("..."), "{error}"); + } + + /// The search for an answer among other output reads a bounded number of + /// pieces: an answer just inside the bound is found and named, one just + /// past it is not looked for, and the stdout is described as a whole. + #[test] + fn an_answer_past_the_search_bound_is_not_looked_for() { + let envelope = r#"{"ok":true,"data":{}}"#; + let inside = format!("{}{envelope}\n", "log\n".repeat(STDOUT_SEARCH_PIECES - 1)); + assert_eq!( + answer_among_other_output(inside.as_bytes()), + Some(OtherOutput::BeforeAnswer("log".to_owned())) + ); + let past = format!("{}{envelope}\n", "log\n".repeat(STDOUT_SEARCH_PIECES)); + assert_eq!(answer_among_other_output(past.as_bytes()), None); + assert!( + unreadable_stdout(past.as_bytes()).starts_with("the engine's stdout is not JSON"), + "{}", + unreadable_stdout(past.as_bytes()) + ); + } + + /// The quoted part of an unreadable stdout is its first non-blank line, + /// bounded, so a megabyte of output does not become a megabyte of error. + #[test] + fn the_stdout_excerpt_is_the_first_line_and_bounded() { + assert_eq!(stdout_excerpt(b"\n\n first \nsecond"), "first"); + let long = "x".repeat(500); + let excerpt = stdout_excerpt(long.as_bytes()); + assert_eq!(excerpt, format!("{}...", "x".repeat(STDOUT_EXCERPT_CHARS))); + let exact = "y".repeat(STDOUT_EXCERPT_CHARS); + assert_eq!(stdout_excerpt(exact.as_bytes()), exact); + } + + /// The cut is made in characters, not bytes, so a multi-byte character that + /// straddles the byte position of the cut is kept whole or left out whole, + /// and a control character in a decoder's message is escaped. + #[test] + fn bounded_cuts_in_characters_and_escapes_control_characters() { + for wide in ['\u{e9}', '\u{20ac}', '\u{1f600}'] { + // One ASCII character, then wide ones: in bytes, the cut falls + // inside a wide character; in characters, it falls between two. + let text: String = std::iter::once('a') + .chain(std::iter::repeat_n(wide, STDOUT_EXCERPT_CHARS)) + .collect(); + let kept: String = text.chars().take(STDOUT_EXCERPT_CHARS).collect(); + assert_eq!(bounded(&text), format!("{kept}..."), "{}", wide.len_utf8()); + assert_eq!(kept.chars().count(), STDOUT_EXCERPT_CHARS); + } + assert_eq!( + bounded("invalid type: string \"a\u{1b}[31mb\", expected a boolean"), + "invalid type: string \"a\\u{1b}[31mb\", expected a boolean" ); } @@ -1164,6 +1905,64 @@ mod tests { assert_eq!(collected, b"still kept\n"); } + /// The capabilities child gets the request's environment and nothing that + /// is the request's own: not its progress request, and not the note on a + /// failing exit, which is about an answer the caller uses. Pure, so it runs + /// on every platform. + #[test] + fn the_check_keeps_the_requests_environment_and_drops_its_progress_and_note() { + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let options = EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: true, + request_progress: true, + forward_stderr: true, + note_failing_exit_after_answer: true, + }; + let check = handshake_options(&options); + assert!( + check + .app_paths + .is_some_and(|kept| std::ptr::eq(kept, std::ptr::addr_of!(paths))) + ); + assert_eq!(check.env_root, options.env_root); + assert_eq!(check.endpoint_key_file, options.endpoint_key_file); + assert!(check.withhold_inherited_key_file); + assert!(!check.request_progress); + assert!(!check.note_failing_exit_after_answer); + } + + /// Withholding an inherited key file holds for the check as well: a + /// third-party engine is not handed a credential path by the request + /// that comes first. + #[test] + fn the_check_withholds_an_inherited_key_file_as_the_request_does() { + let options = EngineStdioOptions { + withhold_inherited_key_file: true, + ..EngineStdioOptions::default() + }; + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &handshake_options(&options), + ); + let key = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); + assert_eq!( + command + .get_envs() + .find(|(name, _)| *name == key) + .map(|(_, value)| value), + Some(None), + "the check must clear an inherited key file, as the request does" + ); + } + #[cfg(unix)] mod child { use super::*; @@ -1193,7 +1992,10 @@ mod tests { assert_eq!(engine.argv(), ["stdio"]); assert_eq!( engine.stdin(), - "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + format!( + "{{\"protocol_version\":\"{ENGINE_PROTOCOL_VERSION}\",\"method\":\"stop\",\ + \"payload\":{{\"service_id\":\"svc\"}}}}\n" + ) ); } @@ -1281,7 +2083,7 @@ mod tests { /// answer, so a caller that asks for it (the daemon) gets a note on this /// process's stderr, and one that does not (the CLI) gets nothing. Run in /// a re-executed test binary so the parent can read that stderr; the - /// decision itself is unchanged: the answer is used. + /// decision itself is unchanged: the answer is read. #[test] fn a_failing_exit_after_an_answer_is_noted_only_when_asked_for() { if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_ARM") { @@ -1306,14 +2108,14 @@ mod tests { .unwrap(); assert!(output.status.success(), "{arm}: {:?}", output.status); let stderr = String::from_utf8_lossy(&output.stderr); - let note = "engine stdio process exited with code 7 after printing its answer; the answer was used"; + let note = "engine stdio process exited with code 7 after printing its answer; the answer was read"; assert_eq!(stderr.contains(note), expected, "{arm}:\n{stderr}"); } } /// The note is only made for an answer that parsed: a child that prints /// something unreadable and fails is reported by the error (with its - /// status), and must not also log that "the answer was used". Both cases + /// status), and must not also log that "the answer was read". Both cases /// assert the outcome and the stderr together, in a re-executed binary. #[test] fn the_note_is_made_only_for_an_answer_that_parsed() { @@ -1336,7 +2138,10 @@ mod tests { assert_eq!( outcome.unwrap_err().to_string(), "failed to parse engine response envelope \ - (engine stdio process exited with code 7)" + (engine stdio process exited with code 7): the engine's stdout \ + is not JSON (expected ident at line 1 column 2), starting with \ + `not json` (an engine writes only its JSON answer to stdout; logs \ + go to stderr)" ); } return; @@ -1354,7 +2159,7 @@ mod tests { assert!(output.status.success(), "{arm}: {:?}", output.status); let stderr = String::from_utf8_lossy(&output.stderr); assert_eq!( - stderr.contains("the answer was used"), + stderr.contains("the answer was read"), expected, "{arm}:\n{stderr}" ); @@ -1481,14 +2286,18 @@ mod tests { let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); assert_eq!( error.to_string(), - "failed to parse engine response envelope; stderr: oops" + "failed to parse engine response envelope: the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` (an engine writes only its JSON answer to stdout; \ + logs go to stderr); stderr: oops" ); let silent = FakeEngine::new("client-garbage-quiet", true, "printf 'not json'"); let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); assert_eq!( error.to_string(), - "failed to parse engine response envelope" + "failed to parse engine response envelope: the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` (an engine writes only its JSON answer to stdout; \ + logs go to stderr)" ); } @@ -1513,7 +2322,10 @@ mod tests { assert_eq!( error.to_string(), "failed to parse engine response envelope \ - (engine stdio process was terminated by signal 9)" + (engine stdio process was terminated by signal 9): the engine's stdout is \ + not JSON (EOF while parsing an object at line 1 column 5), starting with \ + `{\"ok\"` (an engine writes only its JSON answer to stdout; logs go to \ + stderr)" ); } @@ -1534,7 +2346,9 @@ mod tests { assert_eq!( error.to_string(), "failed to parse engine response envelope \ - (engine stdio process exited with code 4); stderr: oops" + (engine stdio process exited with code 4): the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` (an engine writes only its JSON answer to stdout; \ + logs go to stderr); stderr: oops" ); let silent = FakeEngine::new( @@ -1546,14 +2360,18 @@ mod tests { assert_eq!( error.to_string(), "failed to parse engine response envelope \ - (engine stdio process exited with code 5)" + (engine stdio process exited with code 5): the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` (an engine writes only its JSON answer to stdout; \ + logs go to stderr)" ); let clean = FakeEngine::new("client-garbage-clean", true, "printf 'not json'\nexit 0"); let error = run(&clean, &EngineStdioOptions::default()).unwrap_err(); assert_eq!( error.to_string(), - "failed to parse engine response envelope" + "failed to parse engine response envelope: the engine's stdout is not JSON (expected ident at line 1 column 2), \ + starting with `not json` (an engine writes only its JSON answer to stdout; \ + logs go to stderr)" ); } @@ -2048,5 +2866,525 @@ mod tests { ); assert!(status.success(), "{status:?}"); } + + /// What a well-behaved engine answers an `install` with, in this protocol. + fn installed() -> String { + format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":true,"data":{{"installed":true}}}}"# + ) + } + + /// A fake engine that answers `capabilities` by running + /// `capabilities_body` (shell; it must exit) and anything else with + /// [`installed`]. + fn engine_answering_capabilities_with(label: &str, capabilities_body: &str) -> FakeEngine { + let installed = installed(); + FakeEngine::new( + label, + true, + &format!( + "if grep -q '\"method\":\"capabilities\"' \"$dir/stdin\"; then\n\ + {capabilities_body}\n\ + fi\n\ + printf '%s' '{installed}'" + ), + ) + } + + /// Only the install's own stderr reaches this process's stderr: the + /// capabilities child that goes first is not forwarded, even when the + /// caller asks for the install's stderr. The test binary re-executes + /// itself so the parent can read the stderr the arm writes. + #[test] + fn the_checks_stderr_is_not_forwarded_and_the_requests_is() { + const CHECK_LINE: &str = "capabilities child stderr line"; + const INSTALL_LINE: &str = "install child stderr line"; + if std::env::var_os("ROCM_TEST_CLIENT_CHECK_STDERR").is_some() { + let engine = engine_answering_capabilities_with( + "check-stderr", + &format!( + "echo '{CHECK_LINE}' >&2\nprintf '%s' '{}'\nexit 0", + installed() + ), + ); + let body = std::fs::read_to_string(engine.path()).unwrap(); + let body = body.replacen("fi\n", &format!("fi\necho '{INSTALL_LINE}' >&2\n"), 1); + rocm_core::fake_executable::write_fake_executable(engine.path(), body).unwrap(); + let options = EngineStdioOptions { + forward_stderr: true, + ..EngineStdioOptions::default() + }; + let envelope = + engine_request_envelope(EngineMethod::Install, &json!({"runtime_id": "r"})) + .unwrap(); + run_engine_stdio(engine.path(), ["stdio"], &options, &envelope).unwrap(); + assert_eq!(engine.methods(), ["capabilities", "install"]); + return; + } + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_checks_stderr_is_not_forwarded_and_the_requests_is", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_CHECK_STDERR", "1") + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{:?}\n{stderr}", output.status); + assert!(stderr.contains(INSTALL_LINE), "{stderr}"); + assert!(!stderr.contains(CHECK_LINE), "{stderr}"); + } + + fn send(engine: &FakeEngine, method: EngineMethod) -> Result { + let envelope = engine_request_envelope(method, &json!({"runtime_id": "r"})).unwrap(); + decode_engine_response(run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + )?) + } + + /// The capabilities answer that lets the request through: this + /// protocol, a newer one with only additions, or none at all (an engine + /// older than the field). Each time the engine is run twice, the + /// capabilities request first, and the install or launch reaches it. + #[test] + fn a_compatible_engine_is_checked_and_then_sent_the_request() { + let same = format!( + r#"{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":true,"data":{{}}}}"# + ); + for (row, capabilities) in [ + ("same", same.as_str()), + ( + "additive", + r#"{"protocol_version":"0.1.9","ok":true,"data":{}}"#, + ), + ("legacy", r#"{"ok":true,"data":{}}"#), + ] { + for method in [EngineMethod::Install, EngineMethod::Launch] { + let engine = engine_answering_capabilities_with( + &format!("handshake-ok-{row}"), + &format!("printf '%s' '{capabilities}'\nexit 0"), + ); + let answer = send(&engine, method.clone()) + .unwrap_or_else(|error| panic!("{row} {method:?}: {error:#}")); + assert_eq!(answer, json!({"installed": true}), "{row}"); + assert_eq!( + engine.methods(), + ["capabilities", method.name()], + "{row}: the request must follow the check" + ); + assert_eq!( + engine.requests()[0]["protocol_version"], + ENGINE_PROTOCOL_VERSION, + "{row}: the check is itself a versioned request" + ); + assert_eq!( + engine.requests()[0]["payload"], + json!({}), + "{row}: the check sends an empty object, which any engine decodes" + ); + } + } + } + + /// The capabilities child runs on the real request's environment, so the + /// engine checked is the one the request reaches (same app roots, same + /// engine environments root, same endpoint key file), but it is not asked + /// for the install's progress. + #[test] + fn the_check_runs_on_the_requests_environment_without_its_progress() { + let engine = engine_answering_capabilities_with( + "handshake-env", + "printf '%s' '{\"ok\":true,\"data\":{}}'\nexit 0", + ); + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let options = EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: true, + request_progress: true, + forward_stderr: false, + note_failing_exit_after_answer: false, + }; + let envelope = + engine_request_envelope(EngineMethod::Install, &json!({"runtime_id": "r"})) + .unwrap(); + run_engine_stdio(engine.path(), ["stdio"], &options, &envelope).unwrap(); + assert_eq!(engine.methods(), ["capabilities", "install"]); + assert_eq!( + engine.argv_of_run(0), + ["stdio"], + "the check gets the request's args" + ); + assert_eq!(engine.argv_of_run(1), ["stdio"]); + for key in [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + "ROCM_CLI_ENGINE_ENVS_ROOT", + "ROCM_SERVE_API_KEY_FILE", + ] { + assert!( + engine.env_of_run(1, key).is_some(), + "{key} must reach the request" + ); + assert_eq!( + engine.env_of_run(0, key), + engine.env_of_run(1, key), + "{key}: the check must run where the request does" + ); + } + assert_eq!( + engine.env_of_run(1, ENGINE_PROGRESS_STDERR_ENV).as_deref(), + Some("1") + ); + assert_eq!(engine.env_of_run(0, ENGINE_PROGRESS_STDERR_ENV), None); + } + + /// Every way the check can fail stops the request before it is sent: + /// the error says it was not sent and why, naming both versions where + /// the engine named one, and the engine was only ever asked for its + /// capabilities. + #[test] + fn a_handshaked_request_is_not_sent_when_the_check_fails() { + const NOT_SENT: &str = "the install request was not sent to the engine"; + let unchecked = format!( + "the install request was not sent to the engine: its protocol could not be \ + checked against this release's (engine protocol {ENGINE_PROTOCOL_VERSION}) because" + ); + let rows: [(&str, String, String); 11] = [ + ( + "incompatible", + r#"printf '%s' '{"protocol_version":"9.0.0","ok":true,"data":{}}'"#.to_owned(), + format!( + "{NOT_SENT}: the engine speaks engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + ), + ( + "incompatible-refusal", + r#"printf '%s' '{"protocol_version":"0.2.0","ok":false,"error":{"code":"incompatible_protocol","message":"no"}}'"# + .to_owned(), + format!( + "{NOT_SENT}: the engine speaks engine protocol 0.2.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + ), + ( + "unreadable-version", + r#"printf '%s' '{"protocol_version":"banana","ok":true,"data":{}}'"#.to_owned(), + format!( + "{NOT_SENT}: the engine named its protocol version \"banana\", which is \ + not a MAJOR.MINOR.PATCH version, so this release (engine protocol {ENGINE_PROTOCOL_VERSION}) \ + cannot tell whether it can use it" + ), + ), + ( + "incompatible-malformed", + r#"printf '%s' '{"protocol_version":"9.0.0","status":"ok"}'"#.to_owned(), + format!( + "{NOT_SENT}: the engine speaks engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + ), + ( + // The check's refusal on its answer carries no exit note + // and no stderr, even when the child failed and wrote some. + "incompatible-malformed-failing", + r#"printf '%s' '{"protocol_version":"9.0.0","status":"ok"}' +echo oops >&2 +exit 1"# + .to_owned(), + format!( + "{NOT_SENT}: the engine speaks engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + ), + ( + "unreadable-not-text", + r#"printf '%s' '{"protocol_version":9,"ok":true,"data":{}}'"#.to_owned(), + format!( + "{NOT_SENT}: the engine named its protocol version \"9\", which is \ + not a MAJOR.MINOR.PATCH version, so this release (engine protocol {ENGINE_PROTOCOL_VERSION}) \ + cannot tell whether it can use it" + ), + ), + ( + "refused", + format!( + r#"printf '%s' '{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":false,"error":{{"code":"broken","message":"no capabilities today"}}}}'"# + ), + format!( + "{unchecked} it refused the capabilities request: broken: no capabilities \ + today" + ), + ), + ( + "crashed", + "echo 'engine exploded' >&2\nexit 3".to_owned(), + format!( + "{unchecked} asking for its capabilities failed: engine stdio process \ + exited with code 3: engine exploded" + ), + ), + ( + "silent", + "exit 0".to_owned(), + format!( + "{unchecked} asking for its capabilities failed: failed to parse engine \ + response envelope: the engine printed no answer on stdout" + ), + ), + ( + "contaminated", + format!( + r#"echo 'INFO starting' +printf '%s' '{{"protocol_version":"{ENGINE_PROTOCOL_VERSION}","ok":true,"data":{{}}}}'"# + ), + format!( + "{unchecked} asking for its capabilities failed: failed to parse engine \ + response envelope: the engine wrote other output to stdout before its \ + answer, starting with `INFO starting` (an engine writes only its JSON \ + answer to stdout; logs go to stderr)" + ), + ), + ( + "not-json", + "printf 'not json'".to_owned(), + format!( + "{unchecked} asking for its capabilities failed: failed to parse engine \ + response envelope: the engine's stdout is not JSON (expected ident at \ + line 1 column 2), starting with `not json` (an engine writes only its \ + JSON answer to stdout; logs go to stderr)" + ), + ), + ]; + for (row, capabilities_body, expected) in rows { + let engine = engine_answering_capabilities_with( + &format!("handshake-refused-{row}"), + &format!("{capabilities_body}\nexit 0"), + ); + let error = send(&engine, EngineMethod::Install).unwrap_err(); + assert_eq!(format!("{error:#}"), expected, "{row}"); + assert_eq!( + engine.methods(), + ["capabilities"], + "{row}: the install must never reach the engine" + ); + } + + // A refused launch is named as a launch. + let engine = engine_answering_capabilities_with( + "handshake-refused-launch", + r#"printf '%s' '{"protocol_version":"9.0.0","ok":true,"data":{}}' +exit 0"#, + ); + let error = send(&engine, EngineMethod::Launch).unwrap_err(); + assert_eq!( + format!("{error:#}"), + format!( + "the launch request was not sent to the engine: the engine speaks engine \ + protocol 9.0.0, which this release (engine protocol {ENGINE_PROTOCOL_VERSION}) \ + cannot use" + ) + ); + assert_eq!(engine.methods(), ["capabilities"]); + + // An engine that cannot be started at all: the cause the OS gave is in + // the error, not only the client's own context line. + let missing = std::env::temp_dir().join("rocm-no-such-engine-binary"); + let envelope = + engine_request_envelope(EngineMethod::Install, &json!({"runtime_id": "r"})) + .unwrap(); + let error = run_engine_stdio( + &missing, + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .unwrap_err() + .to_string(); + assert!( + error.starts_with("the install request was not sent to the engine: its protocol"), + "{error}" + ); + assert!( + error.contains(&format!( + "failed to spawn engine stdio process {}: ", + missing.display() + )) && error.contains("os error"), + "{error}" + ); + } + + /// The client's own reading of stdout reports a malformed answer in + /// another protocol as the mismatch, not as output that breaks the + /// stdout rule. + #[test] + fn a_malformed_answer_in_another_protocol_is_reported_as_the_mismatch() { + let engine = FakeEngine::answering( + "malformed-incompatible", + r#"{"protocol_version":"9.0.0","status":"ok"}"#, + 0, + ); + let error = run(&engine, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(); + assert_eq!( + error, + format!( + "engine answered in engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ) + ); + assert!(!error.contains("logs go to stderr"), "{error}"); + + // A child that also failed and complained on stderr: the error + // carries both, as for any other unusable answer. + let failing = FakeEngine::new( + "malformed-incompatible-failing", + true, + "printf '%s' '{\"protocol_version\":\"9.0.0\",\"status\":\"ok\"}'\n\ + echo 'engine exploded' >&2\nexit 1", + ); + let error = run(&failing, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(); + assert_eq!( + error, + format!( + "engine answered in engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use \ + (engine stdio process exited with code 1); stderr: engine exploded" + ) + ); + } + + /// The requests that are not checked first (`stop` included) are sent as they are, + /// once, and their answer's version is what is checked. + #[test] + fn a_request_without_a_handshake_is_sent_once() { + for method in [ + EngineMethod::Stop, + EngineMethod::Healthcheck, + EngineMethod::Detect, + EngineMethod::ResolveModel, + EngineMethod::Endpoint, + EngineMethod::Logs, + EngineMethod::Capabilities, + ] { + let engine = FakeEngine::answering("no-handshake", &installed(), 0); + send(&engine, method.clone()).unwrap(); + assert_eq!(engine.methods(), [method.name()], "{method:?}"); + } + let engine = FakeEngine::answering( + "no-handshake-incompatible", + r#"{"protocol_version":"9.0.0","ok":true,"data":{"stopped":true}}"#, + 0, + ); + let error = send(&engine, EngineMethod::Stop).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "engine answered in engine protocol 9.0.0, which this release \ + (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ) + ); + assert_eq!(engine.methods(), ["stop"]); + } + + /// What an engine leaves on stdout besides its answer makes the answer + /// unreadable, and the error says so; a pretty-printed answer over many + /// lines is still one answer. + #[test] + fn output_besides_the_answer_on_stdout_is_refused_and_named() { + let rule = "(an engine writes only its JSON answer to stdout; logs go to stderr)"; + let logged = FakeEngine::new( + "contaminated-before", + true, + &format!("echo 'INFO loading model'\nprintf '%s' '{STOPPED}'"), + ); + assert_eq!( + run(&logged, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(), + format!( + "failed to parse engine response envelope: the engine wrote other output to \ + stdout before its answer, starting with `INFO loading model` {rule}" + ) + ); + + let trailing = FakeEngine::new( + "contaminated-after", + true, + &format!("printf '%s\\n' '{STOPPED}'\necho done >&1\necho oops >&2"), + ); + assert_eq!( + run(&trailing, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(), + format!( + "failed to parse engine response envelope: the engine wrote other output to \ + stdout after its answer, starting with `done` {rule}; stderr: oops" + ) + ); + + let by_position = FakeEngine::answering( + "envelope-by-position", + r#"[null,true,{"stopped":true},null]"#, + 0, + ); + assert_eq!( + run(&by_position, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(), + format!( + "failed to parse engine response envelope: the engine's stdout is JSON but \ + not an answer envelope (an answer is a JSON object), starting with \ + `[null,true,{{\"stopped\":true}},null]` {rule}" + ), + "an array of the envelope's fields is not an answer" + ); + + let not_an_envelope = FakeEngine::answering("not-an-envelope", r#"{"status":"ok"}"#, 0); + assert_eq!( + run(¬_an_envelope, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(), + format!( + "failed to parse engine response envelope: the engine's stdout is JSON but \ + not an answer envelope (missing field `ok` at line 1 column 15), starting \ + with `{{\"status\":\"ok\"}}` {rule}" + ) + ); + + let empty = FakeEngine::new("empty-stdout", true, "exit 0"); + assert_eq!( + run(&empty, &EngineStdioOptions::default()) + .unwrap_err() + .to_string(), + "failed to parse engine response envelope: the engine printed no answer on stdout" + ); + + let pretty = FakeEngine::new( + "pretty-printed", + true, + &format!( + "printf '{{\\n \"protocol_version\": \"{ENGINE_PROTOCOL_VERSION}\",\\n \"ok\": true,\\n \"data\": {{\\n \"stopped\": true\\n }}\\n}}\\n'" + ), + ); + assert_eq!( + run(&pretty, &EngineStdioOptions::default()).unwrap(), + json!({"stopped": true}) + ); + } } } diff --git a/crates/rocm-engine-protocol/tests/protocol_properties.rs b/crates/rocm-engine-protocol/tests/protocol_properties.rs index 8aa54b9971..846bd8504c 100644 --- a/crates/rocm-engine-protocol/tests/protocol_properties.rs +++ b/crates/rocm-engine-protocol/tests/protocol_properties.rs @@ -22,8 +22,9 @@ use rocm_engine_protocol::{ DetectResponse, DevicePolicy, EngineCapabilities, EngineDeviceAvailability, EngineMethod, EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckResponse, - LaunchRequest, LogsRequest, LogsResponse, ResolveModelRequest, - engine_id_from_plugin_binary_name, gpu_indices_to_csv, is_public_bind_host, launch_gpu_indices, + LaunchRequest, LogsRequest, LogsResponse, PRE_VERSIONING_PROTOCOL, ResolveModelRequest, + engine_id_from_plugin_binary_name, engine_protocol_compatible, gpu_indices_to_csv, + is_public_bind_host, launch_gpu_indices, }; use std::collections::BTreeMap; @@ -924,10 +925,81 @@ fn request_and_detect_documents_have_their_documented_wire_shape() { fn failure_envelope_has_its_documented_wire_shape() { assert_eq!( serde_json::to_value(EngineResponseEnvelope::failure("c", "m")).expect("encode"), - serde_json::json!({ "ok": false, "error": { "code": "c", "message": "m" } }) + serde_json::json!({ + "protocol_version": rocm_engine_protocol::ENGINE_PROTOCOL_VERSION, + "ok": false, + "error": { "code": "c", "message": "m" } + }) ); } +/// What a protocol version is, generated: readable `MAJOR.MINOR.PATCH` +/// versions, near misses, and arbitrary text. +fn protocol_version_strategy() -> impl Strategy { + prop_oneof![ + (0_u64..4, 0_u64..4, 0_u64..20).prop_map(|(a, b, c)| format!("{a}.{b}.{c}")), + (0_u64..4, 0_u64..4).prop_map(|(a, b)| format!("{a}.{b}")), + "[ -~]{0,12}", + ] +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] + + /// The version field survives a round trip in both envelopes, whatever it + /// holds, so the reader judges what the writer wrote. + #[test] + fn protocol_version_roundtrips_in_both_envelopes( + version in prop::option::of(protocol_version_strategy()), + ) { + let mut request = EngineRequestEnvelope::new(EngineMethod::Stop, serde_json::json!({})); + request.protocol_version = version.clone(); + let decoded: EngineRequestEnvelope = + serde_json::from_str(&serde_json::to_string(&request).expect("encode")) + .expect("decode"); + prop_assert_eq!(&decoded.protocol_version, &version); + + let mut response = EngineResponseEnvelope::success(1); + response.protocol_version = version.clone(); + let decoded: EngineResponseEnvelope = + serde_json::from_str(&serde_json::to_string(&response).expect("encode")) + .expect("decode"); + prop_assert_eq!(&decoded.protocol_version, &version); + } + + /// A missing version is judged exactly as `0.1.0`, the format from before + /// the field existed, against any version this side might speak. + #[test] + fn a_missing_version_is_read_as_the_pre_versioning_protocol( + ours in protocol_version_strategy(), + ) { + prop_assert_eq!( + engine_protocol_compatible(&ours, None).is_ok(), + engine_protocol_compatible(&ours, Some("0.1.0")).is_ok() + ); + prop_assert_eq!(PRE_VERSIONING_PROTOCOL, "0.1.0"); + } + + /// Every readable version is compatible with itself, and the rule is + /// symmetric between two readable versions. + #[test] + fn compatibility_is_reflexive_and_symmetric( + a in (0_u64..4, 0_u64..4, 0_u64..20), + b in (0_u64..4, 0_u64..4, 0_u64..20), + ) { + let a = format!("{}.{}.{}", a.0, a.1, a.2); + let b = format!("{}.{}.{}", b.0, b.1, b.2); + prop_assert!(engine_protocol_compatible(&a, Some(&a)).is_ok()); + prop_assert_eq!( + engine_protocol_compatible(&a, Some(&b)).is_ok(), + engine_protocol_compatible(&b, Some(&a)).is_ok() + ); + } +} + /// The plugin binary naming rule, positively: the `rocm-engine-` prefix with /// an optional `.exe` in either case yields the id, and anything else none. #[test] diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index d693deb713..a38d55b5e7 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -10,8 +10,9 @@ AIM entries live in [model_catalog.json](../crates/rocm-core/src/model_catalog.j alongside every other model recipe (`"loader": "aim"`). They are hand-maintained and ship on the normal CLI release cadence, through the same PR review as any other recipe. There is no network fetch of the catalog: a hosted, signed -recipe index is blocked on production signing keys (see -[release-trust.md](release-trust.md)) and is out of scope here. +recipe index is blocked on the unpublished recipe index signing key (see +[release-trust.md](release-trust.md#remaining-owner-step)) and is out of scope +here. ## Freshness diff --git a/docs/architecture.md b/docs/architecture.md index 689b1a62a2..d01a70b9ef 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,7 +43,7 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) | The host snapshot behind the internal bridge surface (`build_codex_bridge_snapshot`): examination, config, automation state, managed services, GPUs and the built-in engine list. | relocation | | [`chat.rs`](../apps/rocm/src/chat.rs) | The chat assistant's entry points (`render_chat_text`, `render_chat_prompt_text`, `read_piped_prompt`), the sampling-flag value parsers, and the re-exports other modules reach `chat/` through. | flat | | [`chat/intent.rs`](../apps/rocm/src/chat/intent.rs) | Prompt-intent heuristics: decides from the user's latest message alone which tool call the assistant proposes without the local model (`fallback_rocm_tool_call_for_prompt`) and when an install needs approval. | relocation | -| [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) | The tool dispatch behind `rocm mcp-call`, the dash execution seam and the assistant's read-only tools (`run_internal_mcp_call`), and the in-process runner for read-only `rocm` commands. | relocation | +| [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) | The tool dispatch behind `rocm mcp-call`, the dash execution seam and the assistant's read-only tools (`run_internal_mcp_call`, which runs the argument object exactly as given and never unwraps a nested `arguments` field), the `rocm mcp-call` params-envelope unwrap (`mcp_call_cli_arguments`), and the in-process runner for read-only `rocm` commands. | relocation | | [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs) | Deterministic plain-language summaries of read-only tool output (`deterministic_chat_tool_summary`), shown instead of asking the local model to restate it. | domain | | [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs) | The chat turn loop and transcript (`render_chat_prompt_result_with_progress`): the system prompt, the provider call, running the read-only tools a reply requests, and rendering the reply. | relocation | | [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) | `rocm chat` (`run_chat_command`): hands off to the dashboard chat in an interactive terminal with no `--prompt`, otherwise prints the one-shot reply or the status screen. | relocation | @@ -67,9 +67,9 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`driver_install.rs`](../apps/rocm/src/driver_install.rs) | `rocm install driver` (`install_driver`, `reconcile_driver_install`): runs an approved plan's commands and persists the driver install state. It re-exports `host_install_driver_only_reports` from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) for its callers, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) and [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs). | relocation | | [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) | The `rocm install driver` plan: its model, its printed text, and the per-distro builders (apt, dnf, zypper) from the host's examination and os-release. | domain | | [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) | The WSL2 rocDXG driver plan, which installs the digest-pinned `rocdxg-roct` package or refuses when the download cannot be authenticated. | domain | -| [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) | Per-service inference-endpoint API keys, stored in 0600 files under the services directory, keyed by service id. | relocation | +| [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) | Per-service inference-endpoint API keys, stored in owner-only files under the services directory, keyed by service id. | relocation | | [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) | Which runtime an engine installs into and where its environment lives (`engine_manages_own_runtime`, `env_root_for_engine_install`), shared by the engine and SDK install paths. | relocation | -| [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) | Locating an engine's plugin binary, the engine request entry points (answering from the in-process built-in handler, or running the resolved engine binary with `stdio`), and the in-process built-in engine path. `engine_stdio_child_request` hands the child to the stdio client `rocm` shares with `rocmd`, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal. | relocation | +| [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) | Locating an engine's plugin binary, the engine request entry points (answering from the in-process built-in handler, or running the resolved engine binary with `stdio`), and the in-process built-in engine path, including `rocm __engine-stdio `, which reads its request through the shared `answer_engine_request` so the version is judged before the method. `engine_stdio_child_request` hands the child to the stdio client `rocm` shares with `rocmd`, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal. | relocation | | [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) | Which runtime a `serve` or `engines install` resolves to (`resolve_engine_selection`), the argv and `ROCM_CLI_*` env of a managed child (over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and HTTP readiness polling of a launched service. | relocation | | [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) | `rocm engines` (`engines`), the readiness check `serve` runs for self-managed engines, and the environment root of a running service. | relocation | | [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) | `rocm examine` (`examine`): the human report, or with `--json` the examination plus a summary of what only the CLI knows. | relocation | @@ -105,10 +105,10 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) | Registering runtimes `rocm install sdk` did not install (import, adopt, recovering the `rocm setup` runtime) and the one ready-runtime rule (`select_single_ready_runtime`). | domain | | [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) | Settling an engine onto a runtime after an SDK install (`finish_sdk_install`, `settle_engine_install`): engine-dependency and device checks and the post-install verdict. | relocation | | [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) | Whether an engine install realigns the runtime's torch to the engine's pin or keeps it as an intended divergence, running and reporting that, and the manifest readers it uses. | domain | -| [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) | The installed-runtime registry: the manifest model, where manifests live, writing and loading them, selection and activation validation, and the active-runtime marker. | domain | +| [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) | The installed-runtime registry: the manifest model, where manifests live, writing and loading them, selection and activation validation, and the active-runtime marker with the atomic writer activation, rollback and uninstall use (a failed activation's restore writes or removes it directly, and `rocm config set-default-runtime`/`clear-default-runtime` remove it), and the recorded provenance (`RuntimeProvenance`) that the runtime list and `rocm examine` both print. | domain | | [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) | Reconciling running local servers with a runtime being activated or rolled back: classifying each, the report, and moving stale ones with `--restart-services`. | domain | -| [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) | `rocm runtimes uninstall`: planning, revalidating after the prompt, and applying the removal of a registered runtime, including clearing a Lemonade record inside its folder. | domain | -| [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) | `rocm runtimes` (`runtimes`): activation and rollback with the active-runtime marker write, and the runtime list. | relocation | +| [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) | `rocm runtimes uninstall`: planning, revalidating after the prompt, and applying the removal of a registered runtime, including dropping it as the active or rollback runtime in the config and the active-runtime marker and clearing a Lemonade record inside its folder. | domain | +| [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) | `rocm runtimes` (`runtimes`): activation and rollback, which write the active-runtime marker through `runtime_registry.rs`, and the runtime list (`runtimes_listing`, printed as text or `--json` through `JsonOrText`). | relocation | | [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) | `rocm serve` (`serve`): engine selection (`select_serve_engine`), the AIM pre-flight, and the GPU and device notes folded into the summary. | relocation | | [`serve_port.rs`](../apps/rocm/src/serve_port.rs) | The `rocm serve` port policy: turns an automatic or explicit `--port` request into one leased port before any manifest, log, key or child exists. | domain | | [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) | The pure transforms `serve()` applies to an engine's launch recipe hint: the catalog mapping, then `--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and `--engine-env`. | relocation | @@ -120,11 +120,11 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | | [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. | relocation | -| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the OOM fault-injection seam for end-to-end tests. | relocation | +| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. | relocation | | [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | | [`service_launch/test_support.rs`](../apps/rocm/src/service_launch/test_support.rs) | Test fixtures shared by more than one `service_launch/` child. | flat | -| [`service_records.rs`](../apps/rocm/src/service_records.rs) | Removing a managed-service record and every file it owns, and the `rocm services prune` plan, render and apply steps with their age gate. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. | relocation | +| [`service_records.rs`](../apps/rocm/src/service_records.rs) | Removing a managed-service record and every file it owns — including any copy of its endpoint key an interrupted save left behind, found with one listing of the services folder per command (`staged_endpoint_keys_by_service`) — and the `rocm services prune` plan, render and apply steps with their age gate. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. | relocation | | [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) | `rocm services` (`services`) for `list`, `logs`, `stop`, `restart`, `remove` and `prune`, and the approval-gated stop and restart path. | relocation | | [`services_text.rs`](../apps/rocm/src/services_text.rs) | Log rendering and the `rocm services` and `rocm logs` status, browser and `--follow` surfaces, managed-service record reads, and the `stop_server` answer shape. | relocation | | [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) | `rocm setup` (`setup`): the `setup status` text and `setup reset`, which clears the saved first-time prompt state. | relocation | @@ -154,7 +154,7 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`therock/version.rs`](../apps/rocm/src/therock/version.rs) | Version grammar for TheRock runtimes: the strict install gate (`parse_version`), PEP 440 ordering, host-version parsing and the build-date helpers. | domain | | [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) | The `sdk install` plan a wheel install prints, from one writer for a preview and a real install, resolved or not. | domain | | [`therock_version_proptests.rs`](../apps/rocm/src/therock_version_proptests.rs) | Property tests for version ordering, update verdicts and update-source selection, declared from [`therock.rs`](../apps/rocm/src/therock.rs). | flat | -| [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) | The chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run at once, which need approval, and the validators that refuse unsafe calls. | flat | +| [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) | The chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run at once, which need approval, and the validators that refuse unsafe calls, including any call carrying a nested `arguments` field, so an approval card and the command approval runs cannot read different payloads. | flat | | [`uninstall.rs`](../apps/rocm/src/uninstall.rs) | `rocm uninstall` (`uninstall`): what only uninstall does beyond removing files: undoing the installer's `PATH` change, removing the fish completion script the Linux installer recorded writing (`plan_completion_removals` takes only a regular `completions/rocm.fish` whose first line is the installer's marker naming this install folder, tied to [`install.sh`](../install.sh) by a test; `execute_plan` rechecks the file before removing it and reports a removal failure as a warning instead of stopping, for the `SHELL_COMPLETIONS_KIND` plan entries it adds), and deleting the running `rocm.exe` on Windows. | relocation | | [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) | The uninstall plan shared with `rocm storage`: building it, rendering it for its audience, the `[y/N]` confirmation, and the `remove_path` primitive. | relocation | | [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) | `rocm update` (`update`): the read-only status report, whether to route into preview or apply, applying a runtime update, and the update-source pickers. | relocation | @@ -212,7 +212,7 @@ The `rocmd` binary and the daemon library `rocm daemon` embeds: the watcher loop | --- | --- | --- | | [`lib.rs`](../apps/rocmd/src/lib.rs) | Crate root: module declarations and the entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`, `run_from_args`, `parse_args`). | flat | | [`main.rs`](../apps/rocmd/src/main.rs) | The `rocmd` binary's entry point: a synchronous `main` that calls `rocmd::run_bin_cli()`, which builds the Tokio runtime itself; it holds no logic of its own. | flat | -| [`anthropic_proxy.rs`](../apps/rocmd/src/anthropic_proxy.rs) | The Anthropic-compatible `/v1/messages` front for a served model: a translation proxy to the engine's OpenAI API, translating streaming requests and, for non-streaming ones, tool use both ways, on a port of its own, published in the engine state file (`spawn_alongside_engine`). | flat | +| [`anthropic_proxy.rs`](../apps/rocmd/src/anthropic_proxy.rs) | The Anthropic-compatible `/v1/messages` front for a served model: a translation proxy to the engine's OpenAI API, translating streaming requests and, for non-streaming ones, tool use both ways, on a port of its own, published in the engine state file (`spawn_alongside_engine`). A state file refused as empty or not valid JSON, as one an engine is rewriting in place can be for a moment, is tried again a few times, 200 ms apart; any other failure to record the endpoint is returned at once. | flat | | [`artifact_prefetch.rs`](../apps/rocmd/src/artifact_prefetch.rs) | The `prefetch_artifact` sandbox tool's download: whether a recipe artifact may be fetched under the approved source policy (answering `source_policy_required`, `cached` or `blocked` without touching the network when it may not), the host-scoped Hugging Face URL checks that gate a token, and the streamed, sha256-verified download that records the cache marker. | flat | | [`cli.rs`](../apps/rocmd/src/cli.rs) | The `Cli`/`Command` clap definitions, the bridge-snapshot and pretty-JSON printers, and the top-level dispatch (`run_cli`, `run_bin_cli`, `run_from_args`). | flat | | [`common.rs`](../apps/rocmd/src/common.rs) | Small shared helpers: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`), and argument and JSON-payload utilities such as `payload_string`. | flat | @@ -226,7 +226,7 @@ The `rocmd` binary and the daemon library `rocm daemon` embeds: the watcher loop | [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) | Running a sandbox tool under bubblewrap or natively, the sandbox-tool dispatch (`run_sandbox_tool`) and the result shaping for `check_updates`, `driver_plan` and `stop_server`. For `prefetch_artifact` it resolves the artifact reference against the model recipe registry and hands the artifact to [`artifact_prefetch.rs`](../apps/rocmd/src/artifact_prefetch.rs). | flat | | [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) | The sandbox-tool vocabulary: the `SandboxToolArg` enum and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags. | flat | | [`service.rs`](../apps/rocmd/src/service.rs) | Stopping a managed service: the engine `Stop`, the manifest and endpoint-key writes, and the stop report (`stop_managed_service`, `stop_report_confirmed`). The termination of the recorded processes is the routine shared with `rocm services stop`, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs); this module passes it `terminate_recorded_pid` and keeps no copy of it or of its grace constant. | flat | -| [`supervise.rs`](../apps/rocmd/src/supervise.rs) | The `rocmd supervise` entry point (`supervise_service`: launch the engine child, record it, wait for readiness, record the exit) and the recovery respawn (`restart_managed_service`). | flat | +| [`supervise.rs`](../apps/rocmd/src/supervise.rs) | The `rocmd supervise` entry point (`supervise_service`: launch the engine child, record it, wait for readiness, record the exit) and the recovery respawn (`restart_managed_service`). From the spawn until a successful wait, the private `EngineChild` guard holds the engine: on any earlier return or a panic it terminates and reaps the engine's process tree; a record write that `update_supervised_record` skips is not a return. The guard, and the respawn's cleanup of a supervisor it could not record, terminate through `terminate_and_reap` (injectable as `terminate_and_reap_with`), forced and over the process tree via `rocm_core::terminate_recorded_pid_within` from [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs), and reap with a blocking wait only once the child is known gone. A termination that is not confirmed yields a line saying what may still be running: the guard writes it to stderr (a failed write is dropped, not a panic), and the respawn adds it to the error it returns. The engine's log is truncated for each run and then opened for append. An external kill of the supervisor (a signal, the OOM killer) bypasses the guard: not covered; see ROCMAI-957. | flat | | [`test_support.rs`](../apps/rocmd/src/test_support.rs) | Test-only fixtures shared by the crate's module tests, including the workspace-local scratch `AppPaths` and `RecipeIndexEnv`, the guard that serialises tests over the model recipe registry environment variables. | flat | | [`watchers.rs`](../apps/rocmd/src/watchers.rs) | Root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`) that route each event to its watcher's handler. | flat | | [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs) | The `cache-warm` watcher: checks a requested recipe artifact against the model registry and queues a reviewed prefetch proposal for a known one. | flat | @@ -266,7 +266,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) | Locates the `amd-smi` binary (newest managed SDK runtime first, then legacy locations, then `PATH`) and launches it through `amd_smi_json` after the `/dev/kfd` pre-flight. It reads the managed-runtime registry records from [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs), the data and home directory lookups, `has_usable_amd_gpu`, `host_path` and `is_wsl_host`, and the spawn goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`. | domain | | [`anthropic_endpoint.rs`](../crates/rocm-core/src/anthropic_endpoint.rs) | The Anthropic-compatible `/v1/messages` surface of a managed service: `AnthropicEndpointMode`, its state-file keys, and the readiness check and wait on the published proxy endpoint. | domain | | [`archive_props.rs`](../crates/rocm-core/src/archive_props.rs) | Property-test generator and on-disk oracle for the archive extractors: nothing written outside the extraction root, no escaping links, no unsafe mode bits. | flat | -| [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) | `write_file_atomically`: stages the bytes in a unique sibling temp file and publishes it over the destination in one step: durable as well as atomic, since the temp file is flushed before it is published and, on Unix, the directory after it; on Unix it keeps the replaced file's permission bits, and `write_file_atomically` replaces a symlink's target rather than the link. | domain | +| [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) | `write_file_atomically`: stages the bytes in a unique sibling temp file and publishes it over the destination in one step: durable as well as atomic, since the temp file is flushed before it is published and, on Unix, the directory after it; on Unix it keeps the replaced file's permission bits, and `write_file_atomically` replaces a symlink's target rather than the link. `write_owner_only_file_atomically` is the variant for secrets such as served-endpoint keys: it stages into a file from [`owner_only_file.rs`](../crates/rocm-core/src/owner_only_file.rs) instead of keeping the old permissions, and publishes by rename alone, replacing a symlink rather than following it. `staged_siblings` and `staged_destination_name` recognise the staged files a writer killed before publishing leaves behind — only by the exact `{pid}-{millis}-{attempt}` suffix the writer uses — so a cleanup that removes a key can remove those copies too; `stage_owner_only_file_unpublished` (tests and the `test-support` feature only) produces such a leftover for the tests of those cleanups. | domain | | [`automation.rs`](../crates/rocm-core/src/automation.rs) | Automation-daemon runtime state and its JSONL event, audit and proposal stores, with each store's rotation and compaction policy. | domain | | [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) | Locates the `rocm-engine-` sibling binaries and the `rocm` CLI binary itself. | domain | | [`browser.rs`](../crates/rocm-core/src/browser.rs) | Hands a URL to the user's browser behind the `Opener` trait, so tests can replace the opening. | flat | @@ -289,7 +289,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs) | Free-space preflight checks before large downloads and extractions, and the out-of-space error reporting. | flat | | [`download.rs`](../crates/rocm-core/src/download.rs) | Streaming file downloads over `ureq` with resume, retry with backoff, and size, length and SHA-256 checks before the file is published. | domain | | [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) | Generates the CSPRNG bearer key that guards a managed service's endpoint and checks that it is header-safe. | domain | -| [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) | How far a managed service's OpenAI-compatible endpoint has come (`EndpointReadiness`), the inference-verified latch the engine adapters share, and model-reference matching. | domain | +| [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) | How far a managed service's OpenAI-compatible endpoint has come (`EndpointReadiness`), the inference-verified latch the engine adapters share, model-reference matching, and the engine state-file writers that latch shares with the engines' own writers: `merge_json_state_file` and `write_json_state_file` hold one cross-process lock per state directory (`.json-state.lock`) across the whole write, publish through [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), and refuse to merge into a state file that is empty, not valid JSON or not a JSON object. | domain | | [`examine.rs`](../crates/rocm-core/src/examine.rs) | The `Examination` document behind `rocm examine --json` (the field set the doctor skill's `examine.py` emits, plus a few CLI-only additions) and `Examination::probe`, which runs the probes in order; `Examination::probe_with_summary` probes the host once and hands the examination's kernel release to the summary, so `rocm examine --json` reads the kernel release once. | domain | | [`examine/framework.rs`](../crates/rocm-core/src/examine/framework.rs) | The PyTorch probe (through the active runtime or `PATH`) and the llama.cpp probe. | domain | | [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) | Host environment probes: OS and distro (with the kernel release read through the summary stack's `detect_kernel_version`), the CPU fields recorded from the shared reader in [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs), tracked environment variables, shared memory, container check and kernel log. | domain | @@ -303,11 +303,11 @@ The library every first-party app and engine crate depends on. It holds host exa | [`fake_executable.rs`](../crates/rocm-core/src/fake_executable.rs) | `write_fake_executable`: writes a stand-in script for tests from a short-lived child, so running it does not fail with `ETXTBSY`. It is built into every build, without a `cfg(test)` gate, so dependent crates' tests can call it. | flat | | [`fix.rs`](../crates/rocm-core/src/fix.rs) | Root of `rocm fix`: the dispatch (`apply`) that plans, confirms and runs a catalog remediation, and the re-exports of its children's public names. | flat | | [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) | The closed remediation catalog: `FixRecipe`, the per-platform class table, `RECIPES`, the lookups by fix-id, and the test-only oracles. | flat | -| [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) | The consent decision and `confirm`, the timeouts the runners use, `current_os`, and output helpers that cannot panic on a closed pipe. | flat | +| [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) | The consent decision and `confirm`, the timeouts the runners use, `current_os`, output helpers that cannot panic on a closed pipe, and `run_step`/`Step`, which run a runner's host command on the bounded executor and turn how it ended (exit code, not started, timed out) into the one stderr line and exit code the runner reports, with `CheckIn` saying whether its check command runs in any shell or in PowerShell, and `read_failure`, the line and exit code for a read the runner plans from that did not complete. | flat | | [`fix/manifest.rs`](../crates/rocm-core/src/fix/manifest.rs) | The JSON catalog manifest `rocm fix --json` publishes, derived from `RECIPES` on every call. | flat | | [`fix/options.rs`](../crates/rocm-core/src/fix/options.rs) | The vocabulary of `rocm fix`: the exit codes (`exit`), `FixOptions` and `FixClass`. | flat | | [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) | Rendering the catalog for people: the listing, a recipe's plan and verify lines, the flag summary `rocm diagnose` shares, and the refusal of a diagnosis position as a fix-id. | flat | -| [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) | The host-mutating runners, one per auto-applicable fix, each printing the exact change, honouring `--dry-run` and asking consent first. | flat | +| [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) | The host-mutating runners, one per auto-applicable fix, each printing the exact change, honouring `--dry-run` and asking consent before any write, except the report-only arms (fix-2 on Linux, fix-9 without `--device-index`). On Windows, fix-2, fix-6 and fix-9 (with `--device-index`) first read the env-scope value their plan depends on, and if that read fails they stop with exit 4 before the runner's own preview or any prompt, under `--dry-run` too. | flat | | [`foreign_temp_probe.rs`](../crates/rocm-core/src/foreign_temp_probe.rs) | Test probe that a writer publishing through `atomic_write` never touches another writer's temp file; built only for tests and the `test-support` feature. | flat | | [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. | domain | | [`gguf.rs`](../crates/rocm-core/src/gguf.rs) | Reads a GGUF model's shape (layers, heads, head width, parameter count, data offset) from its header, never the weights. | domain | @@ -326,12 +326,13 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | | [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) | The one routine that reads the last lines of a log file (`tail_lines_decoded`, `tail_log_decoded`): seek-bounded, lossy, with plain or console decoding. | flat | -| [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) | The TheRock managed-runtime environment: `ManagedRuntimeEnvironment`, the active-runtime lookups, `prepend_runtime_paths`, and the registry and SDK probing behind them. | domain | +| [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) | The TheRock managed-runtime environment: `ManagedRuntimeEnvironment`, the active-runtime lookups, `prepend_runtime_paths`, the registry and SDK probing behind them, and the library shims a runtime's processes need on their loader path (`ensure_amd_smi_compat`, `runtime_loader_path_entries`). | domain | | [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) | The curated model catalog: recipe types, the built-in catalog JSON, signed external recipe indexes and artifact cache markers. | domain | | [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) | Whether a model will run on this host, from its curated recipe and the host's measured GPU and RAM, without fetching weights (`rocm diagnose --model`). | domain | | [`net.rs`](../crates/rocm-core/src/net.rs) | The default local host and port, URL formatting and parsing, and the hand-rolled loopback HTTP/TCP client the readiness probes use. | domain | -| [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) | OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans and hints vLLM's ROCm wheels need. | flat | +| [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) | OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim (and the removal of a leftover stub where the host has a real library), and the per-library install plans and hints vLLM's ROCm wheels need. | flat | | [`os_release.rs`](../crates/rocm-core/src/os_release.rs) | The one `/etc/os-release` parser, reading the file as a POSIX shell sourcing it would. | flat | +| [`owner_only_file.rs`](../crates/rocm-core/src/owner_only_file.rs) | `open_new_owner_only`: creates a new file, never opening an existing file or link, that only the current user can open: mode 0600 on Unix, and on Windows a protected DACL granting only the process token's user, applied by `CreateFileW` so nothing is inherited, with the reparse-point flag and long-path (`\\?\`) conversion `std` uses for `create_new`, so a symbolic link is never followed and long paths work. `owner_only_sddl` is that DACL in SDDL, built on every OS; `owner_only_dacl_mismatch` (Windows, tests and the `test-support` feature only) reads a file's DACL back for the tests of the crates that store secrets. | domain | | [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) | Distro-generic package install planning: `PackageManager` resolution, the plan types, the root and passwordless-`sudo` checks, and the `apt-get -s` safety helpers. It never executes a plan; the one process it starts is `sudo -n true`, behind `can_autoinstall`. | flat | | [`paths.rs`](../crates/rocm-core/src/paths.rs) | `AppPaths` (config, data and cache roots and their per-engine and per-service subpaths), the managed-root override and engine plugin directories, and `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, with the constants that name them (`CONFIG_DIR_ENV`, `DATA_DIR_ENV`, `CACHE_DIR_ENV`, `ENGINE_ENVS_ROOT_ENV`). | domain | | [`persisted_paths_props.rs`](../crates/rocm-core/src/persisted_paths_props.rs) | Property test that `RocmCliConfig::load` and `AppPaths::discover` agree about where the data dir is: a `config.json` that cannot be read makes discovery fail or still find the recorded managed root, never silently fall back to the default data dir. Declared from [`paths.rs`](../crates/rocm-core/src/paths.rs). | flat | @@ -348,9 +349,11 @@ The library every first-party app and engine crate depends on. It holds host exa | [`report_trend.rs`](../crates/rocm-core/src/report_trend.rs) | Weekly counts of Doctor reports, and the weeks in which a known problem rose sharply. | domain | | [`rocm_install.rs`](../crates/rocm-core/src/rocm_install.rs) | Discovery of unmanaged ("legacy") ROCm installs: `LegacyRocmSummary`, the install-layout search and version ranking. | domain | | [`runtime.rs`](../crates/rocm-core/src/runtime.rs) | The host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default directories, path helpers and `install_root_has_payload`. | flat | +| [`runtime_rocdxg.rs`](../crates/rocm-core/src/runtime_rocdxg.rs) | Which `librocdxg` a managed wheel runtime's HSA runtime loads on WSL: finds runtimes that ship `librocdxg.so.1` but whose `librocdxg.so` (the name the runtime opens) is absent or a dangling link, so they load the host's copy, links the name for a runtime `rocm install sdk` has just installed (replacing a dangling link atomically; a name that resolves when checked is left alone), and builds the `ln -sfn` command `rocm examine` advises for one installed earlier. | domain | | [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs) | `ServiceId`, the validator every per-service id should pass before it reaches a path builder, and `generate_service_id` for a new launch. | domain | | [`service_record.rs`](../crates/rocm-core/src/service_record.rs) | `ManagedServiceRecord`, the on-disk manifest of a managed inference service, and the engine state-file keys that tie it to readiness probing. | domain | | [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) | `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child, and each write forwarding the child's stderr to its own, in it. Guarded regions may overlap, across threads too; the disposition in force before the first is put back when the last ends. | domain | +| [`tensorizer_override.rs`](../crates/rocm-core/src/tensorizer_override.rs) | The tensorizer release the vLLM install puts over vLLM's own exact pin so vLLM imports on Python 3.14, and the rule that tells that deliberate `uv pip check` finding from a defect, shared by the engine and the CLI. | domain | | [`terminal.rs`](../crates/rocm-core/src/terminal.rs) | One ECMA-48 walk over untrusted terminal output, and the stripping, line-splitting and quotability decisions built on it. | flat | | [`test_env.rs`](../crates/rocm-core/src/test_env.rs) | `RestoredEnvVar`: restores a process environment variable on drop, so a panicking test cannot leak it. | flat | | [`test_support.rs`](../crates/rocm-core/src/test_support.rs) | Test fixtures for throwaway per-test `AppPaths` directories and for seeding the automation audit store. | flat | @@ -380,6 +383,7 @@ Boundaries: - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs)'s `amd_smi_json` is the synchronous launch path for `amd-smi`, and the `/dev/kfd` pre-flight runs there; the dashboard collectors run their own pre-flight through the same bounded `rocm_core::device_readable_bounded` probe, over a `rocm-dash-collectors` to `rocm-core` edge that [`crate_edges.rs`](../xtask/src/crate_edges.rs) allows; [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) fails any unlisted caller of `resolve_amd_smi_binary`. - [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) is the only log-tail reader behind every log view, and lives here because `rocmd` cannot depend on `rocm`. - A new atomic file write goes through [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs); a few older temp-then-rename writers in `apps/rocm` predate it. + - A write of an engine's JSON state file goes through [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs)'s `merge_json_state_file` or `write_json_state_file`, which hold one lock per state directory across the whole read-modify-write, so two merges (an engine wrapper's, a healthcheck's, `rocmd`'s Anthropic endpoint publish) never drop each other's keys; a full rewrite through `write_json_state_file` is ordered by the same lock but still replaces what was merged before it. The Lemonade engine writes only through them, while the vLLM and AIM engines still write their own state files directly and unlocked, so only the shared writers' merges into those files (the healthcheck's and `rocmd`'s Anthropic endpoint publish) take the lock. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s `runtime_python_env_root_for_executable` and `install_root_has_payload` are the one definitions of those rules. - [`vllm_oom.rs`](../crates/rocm-core/src/vllm_oom.rs)'s `VLLM_OOM_DETECTED_PREFIX` and `VLLM_OOM_DIAGNOSE_COMMAND` must match byte for byte in the vLLM engine and `apps/rocm`. - **Dependency direction.** [`automation.rs`](../crates/rocm-core/src/automation.rs) reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), which imports nothing back; the built-in watcher catalog sits beside `WatcherMode` in [`config.rs`](../crates/rocm-core/src/config.rs) to keep it that way. @@ -609,9 +613,10 @@ A contract surface: verify all impacted engines after any protocol change here ( | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) | The protocol contract: protocol versions, the request/response envelopes and per-method types, device and GPU-selection policy, engine-plugin discovery, the endpoint API key file carrier shared by `rocm` and `rocmd`, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. | flat | -| [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) | Test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. | flat | -| [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) | The client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, whether to withhold an inherited key file from a child that is handed none, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). | flat | +| [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) | The protocol contract: protocol versions (`ENGINE_PROTOCOL_VERSION`), the request/response envelopes with their `protocol_version` field and `EngineRequestEnvelope::new` that stamps it, the per-method types and `EngineMethod::requires_protocol_handshake`, device and GPU-selection policy, engine-plugin discovery, the endpoint API key file carrier shared by `rocm` and `rocmd` (including `remove_endpoint_key_files`, which also removes leftover staged copies of a key), and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. | flat | +| [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) | Test-only, behind the `test-support` feature that the `rocm`, `rocmd`, `lemonade`, `vllm` and `aim` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment, stdin, and the environment and request of each of its runs, and answers (or crashes) as a test asks, so both callers' tests run against the same fake. | flat | +| [`protocol_version.rs`](../crates/rocm-engine-protocol/src/protocol_version.rs) | The engine protocol version rule: hand-parsed `MAJOR.MINOR.PATCH`, a missing version read as `0.1.0`, the caret compatibility rule (`engine_protocol_compatible`, `ProtocolMismatch`), the bump policy, `incompatible_request_response` (the `incompatible_protocol` refusal for a decoded request), `answer_engine_request` (an engine's reading of request text: the version judged before the method, then `unknown_method` or `invalid_request` for the other unusable requests), and, behind the `test-support` feature, `engine_contract`, the one protocol check every engine's tests run. | flat | +| [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) | The client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, whether to withhold an inherited key file from a child that is handed none, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as (an empty stdout, other output before or after a top-level answer, quoting that output, JSON that is not an envelope, or not JSON, quoting the start of stdout), the protocol check before an `install` or a `launch` (a `capabilities` request first, on the request's environment without its progress request, and the request is not sent unless its answer is in a compatible protocol; `rocm` answers its built-in engines in-process without it, and no shipped command sends an install or launch to an engine binary today), and `decode_engine_response`, which checks the answer's protocol before anything else in it (an object that does not decode as an envelope has its protocol judged from the raw JSON before its shape is). It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs), and the version rule is [`protocol_version.rs`](../crates/rocm-engine-protocol/src/protocol_version.rs). | flat | ### `crates/rocm-deps` @@ -630,7 +635,7 @@ HTML and markdown reporting for the cucumber E2E suite, consumed only by `xtask` | [`lib.rs`](../crates/e2e-report/src/lib.rs) | Crate root: module declarations and the public API re-exports. | flat | | [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) | The CLI command-coverage audit table: which `rocm` commands a platform's run exercised and whether they passed. | domain | | [`components.rs`](../crates/e2e-report/src/components.rs) | Shared maud HTML fragments plus the CSS and timestamp helpers both report generators use. | flat | -| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, plus the report text both renderers share. | domain | +| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, `platform_status` (one platform's status, public so the harness can test the `platform.json` it writes), plus the report text both renderers share. | domain | | [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) | Artifact-name identity: parses an artifact or directory name into platform, OS and channel, and owns the platform-label rule. | domain | | [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) | The HTML renderer for the consolidated report. It does not render `PlatformVersions::provenance()`: the GFX target and package source appear only in the Markdown output (the wiki page and the step summary that shares its renderer). | domain | | [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) | The markdown renderer, including the table piped into `$GITHUB_STEP_SUMMARY`. | domain | @@ -652,7 +657,7 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../engines/lemonade/src/lib.rs) | Crate root: CLI and envelope dispatch, the service-lifecycle orchestration that calls across the modules, and the crate-root re-exports `apps/rocm` uses. | flat | +| [`lib.rs`](../engines/lemonade/src/lib.rs) | Crate root: CLI and envelope dispatch (the `stdio` request read through the shared `answer_engine_request`, and a request for an incompatible engine protocol refused before its payload is read), the service-lifecycle orchestration that calls across the modules, and the crate-root re-exports `apps/rocm` uses. | flat | | [`main.rs`](../engines/lemonade/src/main.rs) | The `rocm-engine-lemonade` binary's entry point; it only calls `rocm_engine_lemonade::run_cli()`. | flat | | [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) | The cached embeddable-archive layer: SHA-256 verification, verified publication into the cache, and the per-archive lock `ensure_cached_archive` returns as an `ArchiveCacheGuard` to hold while using the bytes. | domain | | [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) | The hardened in-process tar.gz and zip extractor for the embeddable archive, and the containment-checked `copy_tree` into the runtime directory. | domain | @@ -660,12 +665,13 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, the device availability `detect` reports, and the notice that names a non-ROCm GPU backend (Vulkan) as the one serving. | domain | | [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, including a registered model name resolved through the installed Lemonade catalog, which route a name serves by, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | | [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | -| [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | +| [`host_libraries.rs`](../engines/lemonade/src/host_libraries.rs) | Whether this host's glibc and OpenSSL are new enough for the pinned Linux `lemond`: reads the newest `GLIBC_*`/`OPENSSL_*` version node the loader cache's `libc.so.6` and `libssl.so.3` define, or finds that a cache listing `libc.so.6` has no `libssl.so.3`, and refuses before download and before `lemond` starts, naming what is needed and what the host has. | domain | +| [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, the pinned archives' digests with what the Linux `lemond` needs from the host (`LINUX_LEMOND_NEEDS`), plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | | [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) | llama.cpp backend discovery and selection: asks `lemonade backends` what the host supports, picks and installs the best GPU backend, and finds its `llama-server`. | domain | -| [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn, environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | -| [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) | The router serve path: spawn `lemond`, select a GPU backend, load and verify the model by its router name, and on a failure after the spawn stop the server and settle the state. A load or verification failure may then be served directly; a probe that could not reach the endpoint is not one. | domain | +| [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn (refused on Linux for a host older than the pinned build needs), environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | +| [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) | The router serve path: spawn `lemond`, select a GPU backend, load and verify the model by its router name, and on a failure after the spawn stop the server with the bounded `terminate_pid` in [`state.rs`](../engines/lemonade/src/state.rs) and settle the state, except that a state file that is empty, torn or not a JSON object refuses the `failed` mark and is left as it was. A load or verification failure may then be served directly; a probe that could not reach the endpoint is not one. | domain | | [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) | `XDG_RUNTIME_DIR` tier 1, 2 and 3 hardening for the spawned child. | domain | -| [`state.rs`](../engines/lemonade/src/state.rs) | Service state file I/O and port and endpoint helpers. | domain | +| [`state.rs`](../engines/lemonade/src/state.rs) | Service state file I/O, port and endpoint helpers, and `terminate_pid`, the bounded stop of a still-unreaped child (`lemond` or a direct `llama-server`). On Linux it sends `SIGTERM` to the process tree, then `SIGKILL` for anything still running after the grace period, so the caller's `wait` cannot hang on a process that ignores `SIGTERM`; on Windows it terminates the process itself. A stop it cannot confirm is reported on stderr, including one whose process identity could not be confirmed and which was therefore not signalled. Every state write goes through rocm-core's locked, atomic `write_json_state_file`/`merge_json_state_file`, so a merge into an empty, torn or non-object state file is refused and leaves it as it was. | domain | | [`test_http.rs`](../engines/lemonade/src/test_http.rs) | Test-only: a loopback HTTP fixture that answers each request with a chosen status and JSON body and records what it received, plus (Linux) fake `lemond` and `lemonade` executables that log their arguments. | flat | | [`test_support.rs`](../engines/lemonade/src/test_support.rs) | Test-only fixtures shared by the crate's module tests. | flat | @@ -684,12 +690,13 @@ The vLLM engine adapter (`rocm-engine-vllm`): installs vLLM into a managed TheRo | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../engines/vllm/src/lib.rs) | Crate root: CLI and envelope dispatch and the few items shared across the modules, including `verified_torchaudio_release` for the `rocm` binary, a `pub fn` here that delegates to the `pub(crate)` one in [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) (it is not a `pub use`). | flat | +| [`lib.rs`](../engines/vllm/src/lib.rs) | Crate root: CLI and envelope dispatch (the `stdio` request read through the shared `answer_engine_request`, and a request for an incompatible engine protocol refused before its payload is read) and the few items shared across the modules, including `verified_torchaudio_release` for the `rocm` binary, a `pub fn` here that delegates to the `pub(crate)` one in [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) (it is not a `pub use`). | flat | | [`main.rs`](../engines/vllm/src/main.rs) | The `rocm-engine-vllm` binary's entry point; it only calls `rocm_engine_vllm::run_cli()`. | flat | | [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) | The ROCm 10.x discovery table (`VLLM_ROCM_DISCOVER_BUILD_TABLE`, one row per supported `major.minor` line), its lookup and the series matcher. | domain | -| [`install.rs`](../engines/vllm/src/install.rs) | ROCm and vLLM install execution through `uv`, by the static-pin route or the ROCm 10.x discovery route. | domain | -| [`process.rs`](../engines/vllm/src/process.rs) | The resolve-model, launch, serve and stop handlers: recipe validation, device-policy and GPU-index resolution, the vLLM server spawn and launch environment, and readiness polling. | domain | -| [`repair.rs`](../engines/vllm/src/repair.rs) | Whether an already-resolvable managed vLLM environment violates vLLM's pinned dependencies and must be reinstalled (`assess_runtime_repair`), treating the SDK's torch build and a user-chosen torch stack (torch, torchvision, torchaudio, triton) with alignment disabled as intended. | domain | +| [`endpoint_auth.rs`](../engines/vllm/src/endpoint_auth.rs) | The endpoint-key middleware for a keyed launch: writes [`rocm_cli_vllm_auth.py`](../engines/vllm/src/rocm_cli_vllm_auth.py) (compiled in with `include_str!`) under the engine's data directory, prepends that directory to the server's `PYTHONPATH` and passes `--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`, which requires the bearer key on every HTTP and WebSocket route except `GET`/`HEAD /health` and CORS preflights. `VLLM_API_KEY` alone covers only a fixed set of path prefixes. | domain | +| [`install.rs`](../engines/vllm/src/install.rs) | ROCm and vLLM install execution through `uv`, by the static-pin route or the ROCm 10.x discovery route, then a check that vLLM's `tensorizer` imports (upgrading it when it does not). | domain | +| [`process.rs`](../engines/vllm/src/process.rs) | The resolve-model, launch, serve and stop handlers: recipe validation, device-policy and GPU-index resolution, the vLLM server spawn and launch environment, readiness polling, and the bounded teardown of a server that never became ready (`fail_unready_server`: unless the readiness poll already reaped it, `SIGTERM` to the tree escalating to `SIGKILL`; a stop it cannot confirm is reported on stderr and the server itself is killed; then the reap and the `failed` state). | domain | +| [`repair.rs`](../engines/vllm/src/repair.rs) | Whether an already-resolvable managed vLLM environment violates vLLM's pinned dependencies and must be reinstalled (`assess_runtime_repair`), treating the SDK's torch build, a user-chosen torch stack (torch, torchvision, torchaudio, triton) with alignment disabled, and the tensorizer upgrade as intended. | domain | | [`runtime.rs`](../engines/vllm/src/runtime.rs) | TheRock managed-runtime resolution and library and bin path derivation, plus the launch path's compat-shim writes (`ensure_therock_compat_shims`). | domain | | [`startup_diagnostics.rs`](../engines/vllm/src/startup_diagnostics.rs) | What a failed startup tells the user (`startup_log_context`): the tail of the startup log, then the out-of-memory and import-failure hints that point at `rocm diagnose`. | domain | | [`state.rs`](../engines/vllm/src/state.rs) | Service state file I/O, log tailing, and health and inference probing. | domain | @@ -699,7 +706,7 @@ Boundaries: - **One discovery table.** [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) is the single list of the ROCm SDK versions installed by discovery: install routing, the fail-closed static fallback and launch-time environment setup all read its rows, and nothing keeps a second list in step with them. It is a leaf that imports no other module of the crate, which does not keep its readers apart: [`install.rs`](../engines/vllm/src/install.rs) imports [`runtime.rs`](../engines/vllm/src/runtime.rs) and [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs), and all three read the table. - **Layering.** [`install.rs`](../engines/vllm/src/install.rs) reads [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) and [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) and asks [`repair.rs`](../engines/vllm/src/repair.rs) whether an install pass is needed; none of them imports [`install.rs`](../engines/vllm/src/install.rs). -- **Path derivation writes nothing.** [`runtime.rs`](../engines/vllm/src/runtime.rs)'s path derivation performs no filesystem writes; the compat-shim writes are `ensure_therock_compat_shims`, called only from [`process.rs`](../engines/vllm/src/process.rs), and the running-state payload never calls it. +- **Path derivation writes nothing.** [`runtime.rs`](../engines/vllm/src/runtime.rs)'s path derivation performs no filesystem writes; the compat-shim writes are `ensure_therock_compat_shims`, called only from [`process.rs`](../engines/vllm/src/process.rs), and the running-state payload never calls it. The same amd-smi link is also written by `rocm_core::runtime_loader_path_entries` (the install device probe) and by the ComfyUI launch; `rocm examine` only reads it (`runtime_loader_path_entries_read_only`). Each writer also clears a leftover `libmpi_cxx` stub from that directory where the host has a real `libmpi_cxx`, since the directory is then on the loader path. Each writer also removes a `libnuma.so.1` link an older release wrote there, whatever the host has. - **Rendered, not stripped.** [`startup_diagnostics.rs`](../engines/vllm/src/startup_diagnostics.rs) re-renders the log tail line by line through `terminal::rendered_lines` instead of stripping it, because stripping deletes line-break tokens and welds a progress-bar repaint onto the next line. ### `engines/aim` @@ -708,7 +715,7 @@ The AIM engine adapter (`rocm-engine-aim`) runs models as Docker containers inst | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../engines/aim/src/lib.rs) | Crate root: CLI and envelope dispatch, including the `builtin_*` entry points the `rocm` binary calls in-process, the detect and capabilities responses, and the items shared across the modules. | flat | +| [`lib.rs`](../engines/aim/src/lib.rs) | Crate root: CLI and envelope dispatch (the `stdio` request read through the shared `answer_engine_request`, and a request for an incompatible engine protocol refused before its payload is read), including the `builtin_*` entry points the `rocm` binary calls in-process, the detect and capabilities responses, and the items shared across the modules. | flat | | [`main.rs`](../engines/aim/src/main.rs) | The `rocm-engine-aim` binary's entry point; it only calls `rocm_engine_aim::run_cli()`. | flat | | [`docker.rs`](../engines/aim/src/docker.rs) | A thin wrapper around the `docker` CLI binary, with no Docker API client, and `container_name`, a service's deterministic Docker-safe container name. | flat | | [`install.rs`](../engines/aim/src/install.rs) | Install for an engine whose images are self-contained: only a check that Docker is present and reachable. | flat | @@ -741,6 +748,8 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat | | [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat | | [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat | +| [`e2e_prewarm_vision.rs`](../xtask/src/e2e_prewarm_vision.rs) | The pre-warm's torchvision check, split out of [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs): removes a runtime whose torch and torchvision do not match and fails the pre-warm when the active runtime cannot `import torchvision.ops`, unless the default engine does not import torch. | flat | +| [`e2e_queue_import.rs`](../xtask/src/e2e_queue_import.rs) | `cargo xtask e2e-queue-import`: puts a finished shared-queue run's per-lane reports where `e2e-report` reads them, named for the lane and the queue's gate, and writes the queue lanes' status section for the top of the support matrix. | flat | | [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat | | [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat | | [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat | @@ -782,7 +791,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | --- | --- | --- | | [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat | | [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | -| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | +| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, or could not be waited on, non-zero exit, no output, still running at its deadline (killed and reaped), or output left open after exit), keeping the stderr read so far, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | | [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | | [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | | [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | @@ -794,6 +803,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | | [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | +| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no GPU scenario: `@requires-gpu` or `@requires-multi-gpu`) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | @@ -832,7 +842,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | File | Responsibility | Class | | --- | --- | --- | | [`tests/driver_host_guard.rs`](../tests/e2e-cucumber/tests/driver_host_guard.rs) | Ordinary test that the Windows driver-install steps run the host guard before calling the CLI, checked against the step source. | flat | -| [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) | The cucumber test binary: the `E2eWorld` state and its cleanup on drop, the `run_rocm` helpers, the runner `main`, and reconciliation of results with expectations. | flat | +| [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) | The cucumber test binary: the `E2eWorld` state and its cleanup on drop, the `run_rocm` helpers (including `run_rocm_with_scenario_env_off_executor`, which waits for a long CLI run on a blocking thread so it does not stall concurrent scenarios), the runner `main`, and reconciliation of results with expectations. | flat | | [`tests/feature_naming.rs`](../tests/e2e-cucumber/tests/feature_naming.rs) | Drift guard, in the ordinary `cargo test` set, for the `-` scenario names and ids in the `.feature` files. | flat | | [`tests/skill_reference.rs`](../tests/e2e-cucumber/tests/skill_reference.rs) | Drift guard, in the ordinary `cargo test` set, that the catalog table in [`reference.md`](../skills/rocm-doctor/reference.md) uses the vocabulary `rocm fix` prints. | flat | @@ -852,7 +862,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`tests/e2e/diagnose_steps.rs`](../tests/e2e-cucumber/tests/e2e/diagnose_steps.rs) | Steps for [`diagnose.feature`](../tests/e2e-cucumber/features/diagnose.feature): `rocm diagnose` matches and `rocm fix` listings, previews, printed verify commands and failing fixes. | flat | | [`tests/e2e/driver_steps.rs`](../tests/e2e-cucumber/tests/e2e/driver_steps.rs) | Steps for [`driver_install.feature`](../tests/e2e-cucumber/features/driver_install.feature): previewing and approving GPU driver installation plans on WSL, Linux and Windows hosts. | flat | | [`tests/e2e/engines_steps.rs`](../tests/e2e-cucumber/tests/e2e/engines_steps.rs) | Steps for [`engine_shell.feature`](../tests/e2e-cucumber/features/engine_shell.feature): `rocm engines shell` under a pseudo-terminal, since the defect shows only in what the terminal renders. | flat | -| [`tests/e2e/examine_steps.rs`](../tests/e2e-cucumber/tests/e2e/examine_steps.rs) | Steps for [`examine.feature`](../tests/e2e-cucumber/features/examine.feature): `rocm examine`, version and help output, engine listing and the driver plan preview. | flat | +| [`tests/e2e/examine_steps.rs`](../tests/e2e-cucumber/tests/e2e/examine_steps.rs) | Steps for [`examine.feature`](../tests/e2e-cucumber/features/examine.feature): `rocm examine`, the active runtime's provenance, version and help output, engine listing and the driver plan preview. | flat | | [`tests/e2e/host_guard.rs`](../tests/e2e-cucumber/tests/e2e/host_guard.rs) | The host check the Windows driver-install steps run before any CLI call, kept free of the cucumber world so a separate test can include it. | flat | | [`tests/e2e/lemonade_install_steps.rs`](../tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs) | Steps for [`lemonade_installs.feature`](../tests/e2e-cucumber/features/lemonade_installs.feature): what the user is told about the Lemonade copies on disk, from planted trees and install records. | flat | | [`tests/e2e/lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs) | Steps for [`install_lifecycle.feature`](../tests/e2e-cucumber/features/install_lifecycle.feature): `xtask package`, the root installer and the installed binaries, from install through uninstall. | flat | @@ -860,10 +870,11 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`tests/e2e/model_catalog_steps.rs`](../tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs) | Steps for [`model_catalog.feature`](../tests/e2e-cucumber/features/model_catalog.feature): the verbose catalog's AIM entry, its Docker-unavailable flag and its staleness warning. | flat | | [`tests/e2e/planner_steps.rs`](../tests/e2e-cucumber/tests/e2e/planner_steps.rs) | Steps for the planning-assistant scenarios in [`chat.feature`](../tests/e2e-cucumber/features/chat.feature), with a scripted mock proving a request is handed over once and its plan is the one shown. | flat | | [`tests/e2e/remote_steps.rs`](../tests/e2e-cucumber/tests/e2e/remote_steps.rs) | Steps for [`remote.feature`](../tests/e2e-cucumber/features/remote.feature): `rocm remote` against the `fake-tailscale` stand-in and a status document the step writes. | flat | -| [`tests/e2e/runtime_lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs) | Steps for [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` activate, rollback, uninstall and import over planted read-only tarball runtimes. | flat | +| [`tests/e2e/runtime_lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs) | Steps for [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` activate, rollback, uninstall and import over planted read-only tarball runtimes, listing planted nightly wheel, older and adopted runtimes with their provenance in the text and in the JSON, and `runtimes list` as text and as `--json`, including the JSON for a runtime whose folder was deleted and for an active key whose registry entry was, a default runtime set in other letter case through `rocm config set-default-runtime`, an active key edited to other letter case or to a key no runtime has, checked against the active runtime `rocm examine --json` reports, a check that the text markers and JSON flags agree, and a `--json` list that fails on a registry entry it cannot read and prints nothing on stdout. | flat | +| [`tests/e2e/runtime_side_by_side_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs) | Steps for the side-by-side scenarios in [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` list, activate and uninstall over managed wheel runtimes planted as installs from before and after the ROCm 10 layout leave them, comparing their install records byte for byte. | flat | | [`tests/e2e/runtime_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_steps.rs) | Steps for [`runtime_setup.feature`](../tests/e2e-cucumber/features/runtime_setup.feature): SDK install previews and installs, runtime folders reached through a link, and the managed runtime they leave active. | flat | | [`tests/e2e/service_cleanup_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs) | Steps for [`service_record_cleanup.feature`](../tests/e2e-cucumber/features/service_record_cleanup.feature): `rocm services remove` and `prune` over planted service records, asserting on the files that survive, and [`service_record_unqueryable.feature`](../tests/e2e-cucumber/features/service_record_unqueryable.feature): a record whose process cannot be queried (the `e2e-test-hooks` build reads `ROCM_E2E_UNQUERYABLE_PIDS`) is stopped as unverified, refused by `remove`, and cleared by `remove --force` only when no recorded process is confirmed running. | flat | -| [`tests/e2e/service_stop_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_stop_steps.rs) | Steps for [`service_stop.feature`](../tests/e2e-cucumber/features/service_stop.feature) and [`engine_ipc.feature`](../tests/e2e-cucumber/features/engine_ipc.feature): `rocm services stop` and the daemon's `stop_server` tool against a planted service record, and how the daemon reads what an engine child prints, with a stand-in `rocm` staged beside a copy of `rocmd`; and the confirmed stop against a real recorded process, a recorded PID that now belongs to a different process, and a parent that has started a child. | flat | +| [`tests/e2e/service_stop_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_stop_steps.rs) | Steps for [`service_stop.feature`](../tests/e2e-cucumber/features/service_stop.feature) and [`engine_ipc.feature`](../tests/e2e-cucumber/features/engine_ipc.feature): `rocm services stop` and the daemon's `stop_server` tool against a planted service record, and how the daemon reads what an engine child prints (an answer before a failing exit, an unreadable answer, one in an incompatible or a compatible engine protocol, one with a log line before it on stdout), with a stand-in `rocm` staged beside a copy of `rocmd`, and how the real `rocm __engine-stdio vllm` answers requests it cannot serve (another engine protocol, a version that is not text, a method it does not have, input that is not a request); and the confirmed stop against a real recorded process, a recorded PID that now belongs to a different process, and a parent that has started a child. | flat | | [`tests/e2e/service_unknown_id_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_unknown_id_steps.rs) | Steps for [`service_unknown_id.feature`](../tests/e2e-cucumber/features/service_unknown_id.feature): commands that name a local server with no record, checked against the listing they point to. | flat | | [`tests/e2e/serving_steps.rs`](../tests/e2e-cucumber/tests/e2e/serving_steps.rs) | Steps for [`model_serving.feature`](../tests/e2e-cucumber/features/model_serving.feature) and the serve steps other features reuse: `rocm serve` on each engine, ports, managed services and inference. | flat | | [`tests/e2e/skill_steps.rs`](../tests/e2e-cucumber/tests/e2e/skill_steps.rs) | Steps for [`rocm_doctor_skill.feature`](../tests/e2e-cucumber/features/rocm_doctor_skill.feature): drive `rocm` through the sequence the rocm-doctor skill prescribes and check its literal claims. | flat | @@ -873,6 +884,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`tests/e2e/tui_driver.rs`](../tests/e2e-cucumber/tests/e2e/tui_driver.rs) | Pseudo-terminal driver for the dash and chat TUI: spawns the real binary under `portable-pty`, sends keys and reads the emulated `vt100` screen. | flat | | [`tests/e2e/uninstall_steps.rs`](../tests/e2e-cucumber/tests/e2e/uninstall_steps.rs) | Steps for [`uninstall.feature`](../tests/e2e-cucumber/features/uninstall.feature): `rocm uninstall` confined to the scenario root with `--keep-binaries` and redirected caches. | flat | | [`tests/e2e/update_steps.rs`](../tests/e2e-cucumber/tests/e2e/update_steps.rs) | Steps for [`update.feature`](../tests/e2e-cucumber/features/update.feature): the report-only `rocm update`, its feed status, and a loopback catalog where runtimes are planted. | flat | +| [`tests/e2e/vllm_tensorizer_steps.rs`](../tests/e2e-cucumber/tests/e2e/vllm_tensorizer_steps.rs) | Steps for [`vllm_tensorizer.feature`](../tests/e2e-cucumber/features/vllm_tensorizer.feature): a planted Python and `uv` model vLLM's unimportable tensorizer pin, and the install must upgrade it, report it as a deliberate divergence, or refuse. | flat | Boundaries: diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 82f89390b2..d09b6eb6b9 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,32 +37,76 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| | Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | -| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | +| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | | Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | -R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still -covered, but off every PR's critical path, so they moved to the nightly-only -rung instead of being removed outright. +R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to +the nightly-only rung. Their self-hosted nightly jobs, and W7900's, have since +been removed from `nightly.yml`: they never got a runner. The queue (below) is +the scheduled run of R9700 (`r9700s`), MI350P (`mi350p`) and W7900 +(`w7900-dual`). The decommissioning of the runners themselves is separate. -A pull request now runs on exactly one Linux GPU node and one Windows GPU -pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and -the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo +A pull request now runs on one Linux GPU silicon and one Windows GPU pool. +The Linux silicon is MI300X, checked twice: + +- `e2e-gpu-queue`, the primary, dispatches `queue-pr-run.yml` on the shared GPU + queue and publishes the queue run's release report as the `e2e-gpu-report` + cell. +- `e2e-gpu`, the second, runs on the self-hosted MI300X runner and reports as + `e2e-gpu-selfhosted-report`. + +vLLM is the effective engine on both; MI300X is the only silicon where it is. +The Windows pool is the Strix Halo pool (`e2e-gpu-strix-windows` and +`e2e-wsl`). The Strix Halo Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in -the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane -in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue -nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or -dispatch. Its first run, the 11-13 queue nightly -([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. +the merge queue; it keeps its `push`/dispatch triggers and also runs nightly +(`e2e-gpu-nightly-strix` on the same shared Strix pool labels). There is no +W7900 lane in `e2e-selfhosted.yml` any more: W7900 is covered only by the +scheduled queue nightly's `w7900-dual` lane (see +[nightly-on-the-queue.md](nightly-on-the-queue.md)), not on `push`, `release/**` +or dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), +got capacity at once and passed (`Done`, exit 0). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length of a job, and publishes the result; [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes a finished run again. They cover the Linux GPU nightly on both the release and -the nightly channel; native Windows and WSL stay on their dedicated runners, -since the queue runs Linux pods. See +the nightly channel; the Strix Halo lanes stay on the Strix pool, since the +queue has no Strix lane. See [nightly-on-the-queue.md](nightly-on-the-queue.md). +## vLLM route check (advisory) + +[`vllm-route-check.yml`](../.github/workflows/vllm-route-check.yml) runs on a +pull request that touches the resolver that picks the default SDK +([`therock.rs`](../apps/rocm/src/therock.rs) and everything under +[`therock/`](../apps/rocm/src/therock)), the vLLM engine's +[`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) or +[`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) (the build +tables), or the check itself (and on `workflow_dispatch`). On GitHub-hosted +`ubuntu-24.04`, with no GPU and no install, it runs +`rocm install sdk --channel --family --dry-run` +and reports when the selected ROCm SDK version has neither a discovery row nor a +static pin in those tables, matched the way the engine matches them (see +[`vllm_route_check.py`](../scripts/vllm_route_check.py)). Without a route +`rocm engines install vllm` refuses the runtime, and every vLLM-default Instinct +lane runs no scenarios on it: the nightly queue lanes lost their whole nightly +channel to this on the 10.3.0a SDKs. + +It is **advisory and not a required check**. A finding shows as a warning and a +job summary, and the job stays green, so a known gap does not leave a permanent +red X on every pull request that touches those paths. The self-test, the build +and a check that cannot run at all still fail the job: the build tables no +longer parse, or a `rocm install sdk --dry-run` times out, fails or prints no +usable version. It resolves the live indexes, so it can warn on a pull request +that changed nothing, when an index moves to an SDK vLLM has no wheels for; +required, that would block unrelated merges. Read a warning +as "the default SDK has no vLLM build yet": add the discovery row once the +wheels are published, or wait for them. Locally, with a built `rocm`: +`python scripts/vllm_route_check.py --rocm target/release/rocm`. + ## Nightly stream canary [`nightly-canary.yml`](../.github/workflows/nightly-canary.yml) runs daily at @@ -121,20 +165,31 @@ scenario to pass / xfail / skip for that host from its `@id` and `@requires-*` tags, a capability probe, and `expectations.toml` — there is no separate tier flag or tag filter to maintain. -| Job | Workflow | Platform | Runner labels | -|---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | -| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | -| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | -| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | +| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) | +|---|---|---|---|---| +| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | unset | +| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | `1` | +| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | `1` | +| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `1` | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `0` | On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) were cancelled at the 45 min cap; the check is not required and the queue does not wait for it. On a -PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo -pool). +PR the other lanes above run: MI300X (`e2e-gpu`, beside the queue gate +`e2e-gpu-queue` described earlier) and the Windows Strix Halo pool. + +The last column is the lane's declaration of whether it has a GPU. On a lane +that declares `1`, the harness fails before any scenario runs if its capability +probe finds no usable AMD GPU, and after the run if no `@requires-gpu` (or +`@requires-multi-gpu`) scenario executed; on every lane it fails if the probe +could not read the binary under test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes +declare the same way (`0` only for `e2e-wsl-nightly`), as does every lane of +[`run-e2e-lane.sh`](../scripts/run-e2e-lane.sh). `every_self_hosted_lane_declares_whether_it_has_a_gpu` +in [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails when a +self-hosted lane declares neither. See the [E2E README](../tests/e2e-cucumber/README.md)'s +"Lane verdict". `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact @@ -173,8 +228,9 @@ mixed-silicon pool. `every_self_hosted_lane_pins_a_hardware_label` in the lane simply passes on the wrong GPU and reports under the label it was named for. -No workflow other than [`queue-run.yml`](../.github/workflows/queue-run.yml) -and [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) may name the +No workflow other than [`queue-run.yml`](../.github/workflows/queue-run.yml), +[`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) and +[`queue-pr-run.yml`](../.github/workflows/queue-pr-run.yml) may name the `queue-report` label. The shared-queue publishing runner carries that label alone, without GitHub's default `self-hosted`, OS and architecture labels, so a job reaches it only by @@ -185,6 +241,39 @@ against mistakes, not evasion. The runner is registered to this repository alone, so no runner group reaches it. See [nightly-on-the-queue.md](nightly-on-the-queue.md). +`e2e-gpu` starts with a stale-run guard (`pull_request` only). Once the job +has a runner it asks the API for the PR's state and current head. If the PR is +closed, or its head differs from the event's head sha, it records `stale` and +skips every remaining step without failing the job, so the MI300X is free +within seconds. It is a step because a job-level `if` is evaluated before the +job is queued. The guard only reads (`pull-requests: read`); it holds no write +token, because the job runs PR code on a persistent runner. A second job, +`cancel-stale-run`, on a GitHub-hosted runner with no checkout and +`actions: write`, then cancels the run, which frees the PR's concurrency group +and stops its other stale lanes. That follows the split this document already +draws between internal and external contributors: only a same-repo PR is sent a +cancel, while a fork PR (an external contributor, whose hardware lanes are held +for approval where the repository setting requires it, with a read-only token) just has `e2e-gpu` +skip its steps and the rest of its run continue. A failed cancel is tolerated. +The consolidated report does not count a lane that skipped itself as a missing +report, and says so in its summary. `e2e-report` is gated on `always()`, so it +still runs after the cancel, but its missing-report check is gated on +`!cancelled()` and is skipped in a cancelled run. The exemption is therefore for +runs that are not cancelled: a fork PR (no cancel is sent), or a cancel that +fails or loses a race with the check. If the PR cannot be read, the job runs as +before. Only the MI300X lane is guarded: a stale run whose MI300X job already +finished, or never gets to run, keeps its queued Strix jobs until something else +cancels them. A consequence: a re-run of an older run, or of a PR's run after it +merged, cancels itself at this step; to test a specific commit on MI300X, push a +branch or tag at it and use `workflow_dispatch` on that. + +The guard treats any API head that differs from the event's as newer, so a read +that lags a fresh push can end a current run and, for a same-repo PR, cancel all +of it, non-blocking lanes included. To recover, re-run the cancelled run from the +Actions tab ("Re-run all jobs"): by then the API reports the current head, so the +guard passes. If the PR has moved on or merged since, a re-run would cancel +itself again; use `workflow_dispatch` on the branch instead. + `e2e` is the blocking, GitHub-hosted mock job: `@requires-gpu` scenarios resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. @@ -192,8 +281,8 @@ resolve to skip here, and known bugs resolve to xfail from The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as -described below. +scenarios that the mock job cannot. The WSL lane declares no usable GPU +(`E2E_EXPECT_GPU=0`) and its GPU preflight is advisory, as described below. `e2e-wsl` runs on the DevLab Dispatch pool. Its WSL2/Ubuntu-24.04 distro is built fresh inside every job (see above), so there is nothing of its own for @@ -207,9 +296,13 @@ preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite with no hand filtering. It covers WSL host detection, the Windows-to-WSL execution boundary, and whatever GPU access WSL exposes on that machine. The GPU preflight is advisory here precisely because GPU-on-WSL is -what the lane is proving out: where it is unavailable the capability probe -resolves those scenarios to not-applicable and the rest of the suite still -runs. Scenarios the product deliberately routes around on WSL carry +what the lane is proving out. The lane declares `E2E_EXPECT_GPU=0` (forwarded +into the guest through `WSLENV`), because its distro is deliberately left +without the driver bridge (see "What the WSL distro needs"): the capability +probe resolves `@requires-gpu` scenarios to not-applicable, the harness prints +a note saying so rather than failing, and the rest of the suite still runs. If +the probe finds a usable GPU anyway, the harness notes that the declaration is +stale and runs the GPU scenarios. Scenarios the product deliberately routes around on WSL carry `@requires-bare-metal`; scenarios whose premise *is* a WSL host carry `@requires-wsl`, and this is the only lane that runs them. @@ -234,7 +327,9 @@ driver scenarios assert on the plan that command produces — a lane that pre-installed it would hand those scenarios a host already in the state the command under test exists to reach. That costs no coverage today, because `rocm-smi` is absent too, so `@requires-gpu` scenarios resolve to skip either -way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). +way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). This is why +the lane declares `E2E_EXPECT_GPU=0`; a lane that provisions the bridge +declares `1` instead. GPU coverage additionally needs ROCm's WSL passthrough to be complete — `/dev/dxg` and dxcore alone are not enough, `librocdxg.so` must be present and @@ -251,17 +346,30 @@ not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a -pull request runs only the MI300X and Windows Strix Halo pool lanes among them); +pull request runs only the MI300X (queue and self-hosted) and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those -platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from -per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios -included. Each joins its platforms' +platforms (it adds MI300X's `e2e-gpu-nightly` and the Strix Halo Ubuntu, +Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The +R9700, MI350P and W7900 nightlies run on the queue, whose lanes the report reads +separately (next paragraph). Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. Since ROCMAI-429 the report keys each column by `(platform_slug, channel)` rather than platform alone, so `nightly.yml`'s release/nightly channel matrix renders two columns per platform instead of one channel's result silently overwriting the other. +`e2e-report-nightly` also reads the shared queue's nightly (below), which runs +R9700, MI350P, W7900 and the Instinct lanes of the queue's fleet. It fetches the +`queue-run-results` artifact of the newest queue run that tested `main` and +finished, and started, in the last 24 hours, and `cargo xtask e2e-queue-import` +puts each lane's reports among the others, as `MI325X (queue) Linux (release)` +and the like. A lane the queue does not block on is labelled `(queue, advisory)` +and does not count towards the release-readiness verdict; a blocking lane that +came back with no result is shown as `EMPTY` and does. The page opens with a +`Queue lanes` table. A missing or late queue run never fails the report: the +page says no queue lanes are in it. See +[The support matrix and the wiki](nightly-on-the-queue.md#the-support-matrix-and-the-wiki). + On `main`, the `publish-e2e-wiki` job in `nightly.yml` also copies the consolidated nightly matrix to the `E2E-Support-Matrix` page of the repository wiki, overwriting it each night (the page header names the run, commit and @@ -323,21 +431,16 @@ unlike `publish-e2e-wiki`, it still runs (and publishes an explicit outright, instead of leaving the wiki showing a stale verdict with no sign it's stale. -The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, -`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-rad3-report`, -`e2e-gpu-report`, `e2e-gpu-strix-ubuntu-n-1-report`, +The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, +`e2e-gpu-report`, `e2e-gpu-selfhosted-report`, `e2e-gpu-strix-ubuntu-n-1-report`, `e2e-gpu-strix-ubuntu-n-2-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, -`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and +`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` -are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any -more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they -don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is @@ -383,8 +486,9 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and - Strix Halo is `gfx1151` on every OS, so all three Strix lanes (Ubuntu, Windows, WSL2) and their nightly counterparts are lemonade-only — no matrix value turns a Strix lane into a vLLM lane. -- `e2e-gpu` (MI300X) is the only per-PR lane where vLLM is the effective - engine. +- MI300X is the only silicon where vLLM is the effective engine, so the + per-PR vLLM lanes are the two MI300X checks: `e2e-gpu-queue` (on the queue) + and `e2e-gpu` (self-hosted). - Adding a vLLM lane to the default-engine lanes above means adding hardware from a vLLM-eligible family, not adding an `engine:` value to a workflow matrix. A scenario that needs a specific engine regardless of host pins it @@ -403,7 +507,7 @@ has merged) when the workflow's own `changes` job's cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM -lane: they run only on the Windows Strix Halo and WSL2 lanes (both set +silicon (`e2e-gpu-queue` and `e2e-gpu`): they run only on the Windows Strix Halo and WSL2 lanes (both set `E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the `serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X @@ -417,7 +521,9 @@ listed `pull_request` activity type) that joins that PR's group and runs only a no-op job, so it supersedes the PR's earlier run and frees runners where the cancel completes. A run whose job is still queued for a runner may not finish cancelling, so it can keep its place in the queue until the runner picks the -job up. +job up. For a same-repo PR whose MI300X job is that one, its stale-run guard +(see Platforms) ends the job as its first step and a hosted job cancels the run; +the Strix lanes carry no guard. On a release-branch push, every self-hosted GPU job additionally expands into a `strategy.matrix.sdk_version` of `[current, n-1, n-2]` (ROCMAI-430) — three @@ -438,8 +544,6 @@ combinations on that lane: the `nightly` channel is excluded from `n-1`/`n-2`, because it only carries the latest (pre-)release builds and cannot resolve the pinned released versions (`rocm install sdk` fails with "no mutually compatible TheRock ... versions"); those versions are covered by the `release` channel. -`rad3`/`mi350p` are nightly-only (above) and don't run this matrix -at all. `serve` is narrower than `heavy`: it matches only paths that can change serve *behaviour* or the GPU E2E harness (the engines, the @@ -536,6 +640,32 @@ Pre-warm then: - activates the runtime a reuse actually means — after a repair that is the replacement named by `target=`, not the superseded manifest the line belongs to; - ensures the default engine is installed even when the runtime itself is reused; +- removes the active runtime before deciding when its Python cannot `import + torchvision.ops` because torch and torchvision do not match (the + `operator torchvision::nms does not exist` error) and the default engine + imports torch, so the decision that follows installs afresh. It does not do + this for a pre-warm pinned to an exact version, for a runtime the report + marks `ahead_of_index`, or when `rocm update` cannot say which: the runtime + stays, and the check below fails with the manual advice. Any other import + failure is left for the check below. A mismatch that the engine install itself + creates on a reused runtime is found by that check, fails this run, and is + removed at the start of the next one; +- then fails the pre-warm if the active runtime's Python cannot `import + torchvision.ops`, where the default engine imports torch (every engine but + Lemonade; for Lemonade the finding is a warning). A mismatch would otherwise + fail each vLLM serve scenario after its readiness timeout. The probe runs the + runtime's own interpreter, which is where vLLM installs its packages, with the + library directories the runtime's manifest records on the loader path, and is + killed after 120 seconds. The error names the runtime; for a mismatch it says + to run the pre-warm again, which removes the runtime and reinstalls, and for + another failure, or a mismatch the repair declined, it gives the `rocm runtimes uninstall` command with the + pre-warm tree's directories. If the runtime was installed by that same run, + the installer produced the broken pair and the error says so instead. The + check is skipped for a runtime with no torchvision, a tree with no active + runtime or no recorded interpreter, and a shared library that fails to load + when the manifest records no library directories to supply (a warning). An + unreadable `active.json` or registry manifest, a marker without + `runtime_key`, or an interpreter that will not start fails the pre-warm; - reuses the existing tree when it is `up_to_date`, when it is `ahead_of_index` (a pinned build newer than the index must not be rolled back and cannot be reproduced from the index, so it is never offered a repair), or when freshness @@ -580,10 +710,17 @@ it reports. They run with `continue-on-error: false`, so a real regression shows red instead of always green. Their results also surface in the self-hosted consolidated report for visibility. A lane that ran but uploaded no report (it died before writing its results) fails the consolidated report -job, naming the lane, rather than silently dropping out of the grid; on a +job, naming the lane, rather than silently dropping out of the grid (a lane that +skipped itself as stale is the exception); on a `continue-on-error: true` lane that is the only red signal for such a failure. The nightly report job applies the same check to every nightly lane. +A lane the harness failed after its run (no scenario executed, or no GPU +scenario, `@requires-gpu` or `@requires-multi-gpu`, on an `E2E_EXPECT_GPU=1` +lane) records the reason in its `platform.json`, and its row reads FAIL with the reason under "Needs +attention: platforms" (step summary and HTML report alike), even when every +scenario it resolved was a skip. + ### Timeouts on the Strix lanes Every Strix Halo lane raises two budgets rather than letting a slow host read @@ -622,9 +759,14 @@ required list. Self-hosted runners are not used for untrusted fork pull requests: GitHub does not dispatch self-hosted-runner jobs from an external fork's -`pull_request` event without explicit approval. The hardware jobs only run +`pull_request` event without approval (see the repository setting below). The hardware jobs only run against branches and PRs within the repository (and on `workflow_dispatch`, -which requires write access to trigger). +which requires write access to trigger), unless a maintainer approves a fork +run (see below). A same-repo PR's token can be granted write scopes, and only the +GitHub-hosted `cancel-stale-run` job holds one (`actions: write`) in +`e2e-selfhosted.yml`; the self-hosted jobs, which run PR code, hold read scopes +only. Whether GitHub requires approval for every fork run or only a first-time +contributor's is a repository setting, not something these workflows control. That holds for every self-hosted runner here, and so does its limit: approval does not change what an approved fork job may do, and labels do not restrict diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 3cd9610703..0aa8c759a0 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -32,6 +32,100 @@ The `lemonade` adapter uses Lemonade embeddable and prefers Lemonade's `llamacpp:rocm` backend, falling back to `llamacpp:vulkan` when ROCm is unsupported. rocm-cli does not use a CPU fallback for this path. +## The stdio protocol contract + +An adapter run as `rocm-engine- stdio` (and `rocm __engine-stdio +`, which serves the built-in ones) reads one JSON request envelope on +stdin, to its end, and writes one JSON response envelope on stdout. The client +that sends it is [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), +shared by `rocm` and `rocmd`; the version rule is in +[`protocol_version.rs`](../crates/rocm-engine-protocol/src/protocol_version.rs). + +**Every envelope names its protocol.** Requests and responses carry +`protocol_version`, the `MAJOR.MINOR.PATCH` version of the engine protocol the +side that wrote it speaks (`ENGINE_PROTOCOL_VERSION`, `0.1.0` today; `rocm +examine` prints it as `protocol:`). `0.1.0` is the baseline: the release that +introduced the field speaks the same format as the messages written before it. +So an envelope without the field is read as `0.1.0`, and an adapter or a daemon +from an earlier release keeps working with this one. + +**Compatibility is the caret rule.** While the major is `0`, two versions are +compatible when their major and minor match (`0.1.0` and `0.1.9` are, `0.1.0` +and `0.2.0` are not); from `1.0.0` on, when their major matches. The patch never +decides. A version must be three dot-separated plain decimal numbers, each +without a leading zero (`0` itself aside) and fitting in 64 bits; anything else +is unreadable and incompatible with everything. + +**When the version is raised.** + +- An addition an older peer can ignore (a new optional field, a new method + that only an engine ever offers) raises the patch. +- A new method a client may send raises the minor while the major is `0`, and + the major after that, because an older engine cannot decode it. The exception + is a method the client sends only to engines whose capabilities say they have + it. +- A removal or a type change (a field dropped, renamed, made required or given + another type) raises the minor while the major is `0`, and the major after + that. + +**What each side does with a mismatch.** + +- An adapter reads the request's version before anything else in it, its method + included. A request it cannot serve, including one whose `protocol_version` + is present and neither a string nor `null`, gets an error envelope with code + `incompatible_protocol`, whatever its method, and nothing else happens. A + `protocol_version` of `null` is read like a missing one, as `0.1.0`. A + request in a protocol it can serve that names a method it does not have gets + `unknown_method`, and input that is not a request envelope (including JSON + that is not an object) gets `invalid_request`. Each of these is an answer on + stdout. Input that is not valid UTF-8 is not read as a request: the adapter + exits with an error and prints no answer. +- The client checks a response's version before anything else in it, `ok` + and the shape of the rest included, whenever stdout is a single JSON object. + An answer in a protocol it cannot use is an error that names both versions, + even when it reports success or does not otherwise decode. +- Before an `install` or a `launch`, the requests that install or start + something, the client first sends `capabilities` (with an empty object as its + payload) through the same path, on the same environment minus the request + for progress, and checks the version of that answer. The `install` or + `launch` is sent only when that answer is in a compatible protocol and is not + an error; otherwise the error says the request was not sent and why (the + engine's version, or the reason its capabilities could not be read). Nothing + is cached: each such request checks again. The check is made by the shared + client, for an engine binary it runs: `rocm` answers its built-in engines in + its own process, with no check, and no shipped command sends an install or + launch to an engine outside it today. The other methods are sent as they are + and only their answer is checked. `stop` is one of them on purpose: it starts + no extra child for a check first, so a stop is never held behind one. An + adapter that checks the version refuses a `stop` made for a protocol it + cannot serve, before reading its payload. An adapter from before the field + ignores the version and acts on the `stop`; its answer names no version and + reads as `0.1.0`, so the client accepts it while this release's protocol is + `0.1.x` and refuses it once the protocol is past that. When the daemon stops + a server and the engine's answer is refused, the report carries that error + as the engine's result; the daemon + still signals the server's recorded processes itself, so for a `vllm` server + the stop completes, while for an `aim` server, whose stop only the engine can + confirm, it is left unconfirmed. + +**Stdout holds only the answer.** The whole of stdout must parse as one +response envelope, a JSON object, pretty-printed over many lines or not. Anything else there, +such as a log line before the answer, makes the answer unreadable, and the +error says so, quoting the other output (or, when there is no answer at all, +the start of stdout). The rule for an adapter is to write its logs and progress +to stderr, which the client collects and shows when a request fails. The +first-party adapters do not hold it everywhere yet. Known cases where a child +process they start can write to the adapter's stdout: + +- `aim` runs `docker pull` with inherited stdout when it starts a container. +- `lemonade` runs its llama.cpp backend install with inherited stdout. When + it is given no log file, the Lemonade server it starts, the Lemonade CLI it + runs to pull or load a model, and a packaged `llama-server` it runs directly + on a GGUF file all inherit stdout too. +- `vllm` leaves the server's stdout inherited when it is given no log file. + +These are known and not yet changed. + ## Lemonade backend alignment on engine install Lemonade's `resources/backend_versions.json` pins which ROCm SDK version its @@ -54,14 +148,21 @@ The rewrite tries two builds in order. The first keeps Lemonade's own pinned llama.cpp release and just repoints it at the active ROCm version — a deliberate, reproducible pin. If that attempt fails to resolve against the installed backend's shared libraries — for example because that release never -shipped an asset for the active ROCm version — the second queries GitHub's -`releases/latest` for `lemonade-sdk/llama.cpp` (an unauthenticated -`api.github.com` call, subject to GitHub's public rate limit) and installs -whatever build is newest at that moment. That second build is a moving -target, not a pin: which llama.cpp commit actually gets installed depends on -when the install ran, and the version installed is not recorded anywhere -`rocm examine`/`rocm version` report. An attempt that cannot be verified -against the installed backend's shared libraries reverts to whichever version +shipped an asset for the active ROCm version — the second lists the 30 newest +`lemonade-sdk/llama.cpp` releases on GitHub (an unauthenticated +`api.github.com` call, subject to GitHub's public rate limit) and installs the +most recently published one that ships the llama.cpp archive for the active +ROCm version's `major.minor` on this platform +(`llama--bin-ubuntu-rocm--x64.tar.gz` on Linux, +`llama--bin-win-rocm--x64.zip` on Windows). The newest +release alone is not used, because a release can be published without its +ROCm archives. The second attempt is skipped when none of the 30 ships that +archive, when the listing cannot be fetched, and when the build it finds is the +one Lemonade already pins (the first attempt tried that build). That second +build is a moving target, not a pin: which llama.cpp commit actually gets +installed depends on when the install ran, and the version installed is not +recorded anywhere `rocm examine`/`rocm version` report. An attempt that cannot +be verified against the installed backend's shared libraries reverts to whichever version was pinned before the attempt — the packaged pin on a fresh install or after `--reinstall`, but whatever alignment last wrote otherwise — rather than being kept. @@ -132,8 +233,33 @@ The versions of the third-party runtimes rocm-cli downloads are pinned in `runtime-deps.toml` at the repository root — one `[runtime.]` table per runtime. That file is the only place a runtime version is written down: archive names, download URLs, and the dashboard's offline fallback are all -derived from it, so a bump is a one-line edit and the tree cannot end up -holding two different versions of the same runtime. +derived from it, so the tree cannot end up holding two different versions of +the same runtime. A Lemonade bump also records the new release's archive +digests (`EMBEDDABLE_LINUX_SHA256` and `EMBEDDABLE_WINDOWS_SHA256` in +[install.rs](../engines/lemonade/src/install.rs)): the downloaded archive is +checked against them, and a digest cannot be derived from the version. It also +re-reads what the new Linux `lemond` needs from the host, the newest `GLIBC_*` +and `OPENSSL_*` versions `objdump -T lemond` lists, into `LINUX_LEMOND_NEEDS` +beside those digests; a unit test fails until that record names the new Linux +archive. On Linux, `rocm engines install lemonade` refuses a host whose glibc +or OpenSSL is older than that before downloading anything, and the engine +refuses to start that `lemond` on such a host +([host_libraries.rs](../engines/lemonade/src/host_libraries.rs)). For each +library that falls short, the message names the version needed and what the +host has: its version, read from the library the `ldconfig` cache lists, or +that the cache has no such library at all. A host with only OpenSSL 1.1 has no +`libssl.so.3`, so it reads that way. A host whose cache cannot be read, or +whose library does not parse, is not refused. A refused install says what it +left undone: "Nothing was downloaded." for a fresh install, or that the +Lemonade already installed there stays as it is. Because only the `ldconfig` +cache is read, a host that provides OpenSSL 3 or glibc outside it +(`LD_LIBRARY_PATH`, a Nix or Guix profile, a user prefix, a stale cache) is +refused although `lemond` could load there; setting +`ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK` (any value) turns both refusals +into a warning, and the refusal message names it. Lemonade +2026.44.1, the current pin, needs glibc 2.38 and OpenSSL 3.0.0, which Ubuntu +24.04 meets; 2026.45.3's `lemond` needs glibc 2.43 and OpenSSL 3.2.0, which it +does not. `rocm engines list` shows the exact plugin directories for the current host. The same output is available in the TUI with `/engine`. diff --git a/docs/host-probing.md b/docs/host-probing.md index 097a970258..e6b6b3342f 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -306,7 +306,7 @@ One pair per change. Each begins with a commit that adds a characterisation test The later steps of item 6, each its own change and all still open: -- Retire the adapters site by site, as items 2 to 5 remove call sites: a probe that calls `run_bounded` directly reads the typed outcome, so "not installed" stays quiet and "timed out" becomes a `probe_failures` entry without parsing a message, which the adapters cannot do because they keep their return shapes. +- Retire the adapters site by site, as items 2 to 5 remove call sites: a probe that calls `run_bounded` directly reads the typed outcome, so "not installed" stays quiet and "timed out" becomes a `probe_failures` entry without parsing a message, which the adapters cannot do because they keep their return shapes. The `rocm fix` runners' commands and their Windows env-scope reads already do: they run through `run_step` in [fix/consent.rs](../crates/rocm-core/src/fix/consent.rs), so a timed-out step is reported as one, not as exit code `124`. The one other command the runners start, the `id -u` root check, still goes through `run`. - Move `wait_with_output_bounded` in [process.rs](../apps/rocm/src/therock/process.rs) of `apps/rocm` onto the executor, which is the last helper of [#623](https://github.com/ROCm/rocm-cli/issues/623) that still joins its readers with no limit, and decide where `rocmd`'s `parse_sandbox_child_output` quote is bounded. - Make the candidate loop of `capture_optional_command_with_timeout` stop on a timeout instead of starting the next spelling of a program that hung. - Decide whether the runners refuse to resolve a bare program name through an empty `PATH` entry, and whether the adapters take a per-call drain grace: two items of the open list below. diff --git a/docs/json-output.md b/docs/json-output.md new file mode 100644 index 0000000000..f35214b089 --- /dev/null +++ b/docs/json-output.md @@ -0,0 +1,211 @@ + + +# JSON output + +Which `rocm` commands print JSON with `--json`, the rules they follow, and the +commands that do not have it yet. + +## Rules + +- **Text is the default.** `--json` is opt-in per command; leaving it out never + changes the text a person reads. +- **JSON goes to stdout, and only JSON.** With `--json`, stdout holds one JSON + document and nothing else, so a script can parse all of it. Diagnostics, + progress and warnings go to stderr. A command that has no single document to + print is listed below as unsupported rather than given a `--json` that mixes + the two. + + One known exception, tracked as a follow-up: the startup update check that + runs before most commands can print a progress line on stdout ahead of the + document, such as `Python from PATH is not usable for this ROCm install`. + It runs at most once per check interval, and prints only when the runtime it + checks is a wheel install whose recorded Python is not usable and a Python + it finds instead (on `PATH`, or a saved managed one) does not fit. Setting + `ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK` turns the check off, and with it the + line. +- **Errors are not JSON today.** A command that fails with `--json` prints its + error to stderr, prints nothing on stdout, and exits 1, unless the command + documents its own code, as `rocm fix` does. Command lines the argument parser + rejects (an unknown flag, such as `--json` on a command without one, or a + missing argument) exit 2 with a usage message on stderr, before the command + runs. A script tells success from failure by the exit code, not by parsing an + error document. A JSON error shape belongs to the shared envelope below. The + one refusal printed as JSON today is `rocm diagnose --report --json` + declining to prepare a report: that is the command working, so it exits 0 + and prints `refusal_envelope` from + [report.rs](../crates/rocm-core/src/report.rs) (`schema`, `refused`, + `explanation`, `architecture_matrix`). +- **Documented fields do not change silently.** A field may be added. Renaming + or removing one, or giving it a new meaning, needs a versioned transition: + publish the new shape under a new version and keep the old one readable while + callers move. +- **Two shapes are frozen contracts** that other tools are built against: + - `rocm examine --json` — the Examination document the diagnosis tooling + reads. `examination_top_level_keys_match_examine_py_contract` in + [examine.rs](../crates/rocm-core/src/examine.rs) fails if a top-level field + is added, removed or renamed. + - `rocm fix --json` — the fix catalog, carrying `contract_version` + (`CATALOG_CONTRACT_VERSION` in + [manifest.rs](../crates/rocm-core/src/fix/manifest.rs), 1 today). The + rocm-doctor skill states the version it needs, and scenario `skill-07` + holds the two equal. +- **New commands print through the shared helper.** A command whose output is + one value implements `JsonOrText` and calls `print_json_or_text` in + [cli_report.rs](../apps/rocm/src/cli_report.rs), which writes pretty JSON plus + one newline, or the text, and nothing else. + +## Which commands have `--json` + +| Command | Today | Decision | Proven by | +|---|---|---|---| +| `rocm examine` | supported; pretty JSON | supported, frozen | `examine-07`, `examine-08`; `examination_top_level_keys_match_examine_py_contract` | +| `rocm diagnose` | supported; pretty JSON (`DiagnoseOutput` through `JsonOrText`); with `--report`, the prepared report or its refusal | supported | `diagnose-02`, `diagnose-03` | +| `rocm fix` | the catalog only: `--json` is refused with a fix id; `contract_version` 1 | catalog supported, frozen; a JSON result for applying one fix waits for the envelope | `skill-07` | +| `rocm update` | supported for the check; one compact JSON line, not pretty, and refused with `--apply` and `--dry-run` | supported; the compact form is a known difference from the others | `update-02`, `update-03` | +| `rocm storage report` | supported; pretty JSON through `JsonOrText` | supported | `json_or_text_impl_renders_the_report_and_serializes_it` | +| `rocm services list` | supported; pretty JSON. `rocm remote` reads it back over its control channel | supported | `services_json_round_trips_and_applies_the_same_liveness_filter_as_the_table`; exercised indirectly by `remote-15` (`@requires-docker`), whose `rocm remote` reads it back | +| `rocm runtimes list` | supported; pretty JSON through `JsonOrText` | supported, additive-only, no version field yet | `runtime-lifecycle-23` to `runtime-lifecycle-31` and `runtimes_list_json_has_exactly_the_documented_fields_newest_install_first`; what each covers is in [testing.md](testing.md#runtime-selection-and-activation) | +| `rocm runtimes activate`, `rollback`, `uninstall`, `import`, `adopt` | none | follow-up: they change state, so they need the envelope's result and refusal shapes | — | +| `rocm services logs` | none | intentionally unsupported: streamed text, and `-f` never ends | — | +| `rocm services stop`, `restart`, `remove`, `prune` | none | follow-up: needs the envelope, including the refusal path | — | +| `rocm install sdk`, `rocm install driver` | none | follow-up: long-running with progress; needs the envelope and an audit that progress stays off stdout | — | +| `rocm uninstall` | none (`--dry-run` prints the plan as text) | follow-up | — | +| `rocm daemon --status` | none | follow-up; a one-shot snapshot, so a good next command | — | +| `rocm daemon` | none | intentionally unsupported: it runs until stopped | — | + +Commands not in the table (for example `rocm engines list`, `rocm config show`, +`rocm automations list`, `rocm comfyui status`, `rocm setup status`, +`rocm version`) have no `--json` and no decision recorded yet. + +## `rocm runtimes list --json` + +The same listing as the text form, as one object: + +| Field | Value | +|---|---| +| `default_runtime_id` | the configured default runtime id as written in the config, or `null`. The text form labels it `active_runtime_id:`. Not necessarily the active runtime's `runtime_id`: an `active_runtime_key` that names a runtime takes precedence, and the id may differ from its runtime's in letter case or match no runtime at all. Read each runtime's `active` flag to see which one is in use | +| `active_runtime_key` | the configured active runtime key, or `null` | +| `previous_runtime_key` | the configured rollback key, or `null` | +| `registry`, `marker` | paths of the runtime registry folder and the active-runtime marker | +| `active_problem` | `null`, or what is wrong with the configured active runtime, as described under [which runtimes are marked](#which-runtimes-are-marked): `{"kind": "missing_manifest_for_key", "runtime_key"}`, `{"kind": "missing_manifest_for_id", "runtime_id"}`, or `{"kind": "ambiguous_runtime_id", "runtime_id", "runtime_keys": [...]}` | +| `runtimes` | one object per registered runtime, newest install first, ties broken by `runtime_key` | + +Each runtime carries `runtime_key`, `runtime_id`, `version` (as recorded; the +text adds the build date), `format`, `family`, `mode` (`managed` or +`read-only`), `status` (`ready` when the runtime passes the manifest check +that activation applies, `unusable` when it fails it; `ready` does not promise +that an activation succeeds), `unusable_reason` (`null` when ready, otherwise +why the runtime fails that check: text for a person, not a code to match on), +`toolchain` (`included` or `excluded`, the words `rocm examine` uses), `active`, +`rollback_target`, `install_root` and `provenance`. `active` and `rollback_target` are the facts the text marks with +`*` and `-`; a runtime that is both is shown as `*` in the text. The text's +next-step hint for an empty registry is not part of the JSON. + +`provenance` is an object with the four facts the text prints on the runtime's +`provenance:` line, under the same names: `channel`, `source` (the package +index, else the downloaded archive), `source_layout_generation` and +`device_target`. Each is the value the manifest recorded, trimmed, or `null` +where the text prints `unrecorded`: a fact recorded nowhere or recorded blank, +the source of an adopted runtime, the exact target of a tarball or family-wide +install. A script reads `null` as "not recorded"; nothing is guessed from other +facts. + +`registry`, `marker` and `install_root` are strings. A path that is not valid +UTF-8 is written with each invalid sequence replaced by U+FFFD, as the text +form prints it, so such a path cannot be passed back to the filesystem from the +JSON as it stands. + +### Which runtimes are marked + +`active` marks the runtime that `rocm examine`, `rocm runtimes activate`, +`rocm runtimes rollback`, `rocm runtimes uninstall` and `rocm update` treat as +the active one: the listing asks the resolver they use +(`current_runtime_manifest` in +[runtime_registry.rs](../apps/rocm/src/runtime_registry.rs)) rather than +applying rules of its own. That resolver takes the first runtime in list order +whose key matches `active_runtime_key` with ASCII letter case ignored; when +`active_runtime_key` is unset or names no runtime, it takes the one runtime +whose id matches `default_runtime_id`, letter case ignored, and nothing when +that matches none or several. + +`rollback_target` marks the runtime `rocm runtimes rollback` selects: +`previous_runtime_key` resolved the way rollback resolves it +(`select_runtime_manifest`, same file), which ignores surrounding whitespace, +takes an exact key first, then the one key differing only in letter case, then +the one runtime whose id matches. When that selection fails (the value names +nothing, or matches several runtimes), no runtime is the rollback target. A +selected runtime whose `status` is `unusable` is still marked; rollback then +refuses it at the manifest check. + +`active_problem` reports the configuration, by the same matching: + +- `missing_manifest_for_key` when `active_runtime_key` is set and no runtime's + key matches it, letter case ignored. This is the one case where a problem is + reported beside an `active` runtime: when `default_runtime_id` still resolves + one runtime, the commands fall back to it and the listing marks it active, + while `active_problem` still says the configured key names nothing. This + happens when the active runtime's registry entry is deleted outside the CLI + while another registered runtime has the same id (activation records the + active runtime's id as `default_runtime_id`, and installs of one channel and + family share an id). +- With no `active_runtime_key`: `missing_manifest_for_id` when + `default_runtime_id` matches no runtime's id, `ambiguous_runtime_id` when it + matches several (ids differing only in letter case count as several), and + `null` when it matches one, which is then the active runtime. `rocm config + set-default-runtime` reaches this state: it stores the id as typed, clears + `active_runtime_key` and `previous_runtime_key`, and tries to remove the + active-runtime marker file (best effort: a failed removal is ignored). + +Two runtimes can have keys that differ only in letter case on a +case-sensitive filesystem, since `rocm runtimes import` and +`rocm runtimes adopt --runtime-key` store keys as given. For such a pair the +resolver takes the first in list order even when `active_runtime_key` equals +the other exactly, and the listing marks that first one, because it is the one +those commands act on. Each registry entry is a file named `.json`, so on +a case-insensitive filesystem (the default on Windows volumes, and on drvfs +mounts under WSL) the second key names the first one's file: the command +refuses it, or with `--replace` overwrites the first runtime's entry, and this +state cannot occur. + +Not every surface resolves the active runtime this way. Known differences: + +- The runtime manager in `rocm dash` matches exactly: `active_runtime_key` + equal to a runtime's key, or with no `active_runtime_key` the single runtime + whose id equals `default_runtime_id`, and `previous_runtime_key` equal to a + runtime's key. It shows nothing active where the listing marks a runtime + through letter case or through the default-id fallback after a dead key, + shows no rollback target for a `previous_runtime_key` in other letter case, + with surrounding whitespace or given as a runtime id. Of two keys differing + only in letter case it marks the one equal to `active_runtime_key`, where the + listing marks the first in list order, and of two ids differing only in case + it marks the one equal to `default_runtime_id`, where the listing reports + `ambiguous_runtime_id`. +- `rocm setup status` looks `active_runtime_key` up ignoring letter case but + does not fall back to `default_runtime_id`. +- `rocm engines install` without `--runtime-id`, for an engine that runs on the + ROCm runtime, resolves `active_runtime_key` (or, when it is unset, + `default_runtime_id`) the way rollback resolves a key: a dead key fails the + install instead of falling back, and a key matching case twins only by letter + case is refused as ambiguous. + +The shape is additive-only: fields may be added, none will be renamed or +removed without a versioned transition. It carries no version field yet; one +arrives with the envelope if that is adopted. + +## Open decision: a shared envelope + +Every command above prints its own top-level value (an object, or for +`rocm services list` an array). The mutating commands that +are still follow-ups need more than a value: whether the action ran, was +refused or failed, and why, in a shape a script can rely on across commands. A +shared envelope (for example a version, an outcome, the command's result and an +error object) would give them that. + +It is not adopted. Wrapping the existing outputs in it would change documented +shapes, including the two frozen contracts, so it can only arrive alongside +them, under a versioned transition, and the decision belongs with the first +mutating command that needs it. diff --git a/docs/llm-tool-use.md b/docs/llm-tool-use.md index 67667c83c5..7a6cd01b2f 100644 --- a/docs/llm-tool-use.md +++ b/docs/llm-tool-use.md @@ -16,6 +16,10 @@ rocm-cli local assistants use structured tools, not shell commands. model. - Installs, starts, stops, updates, deletes, settings changes, and other mutations must pause on the TUI approval screen. +- The approval card and the command its approval runs are built from the same + argument object. No tool takes a parameter named `arguments`, so a tool call + whose arguments carry a nested `arguments` field is refused before any card is + shown. - The model should inspect first, then request a change only when the inspection result shows it is useful. - The model must use argv-style `rocm` arguments. It must not invent PowerShell, diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f295a2a0ba..4feab8755c 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -24,6 +24,12 @@ This writes `target/release/rocm.exe` on Windows and `target/release/rocm` on Linux/WSL. Run the binary directly on each platform. On WSL/Linux the examine output must report `os: linux` and `wsl: true`. +On WSL with a managed wheel runtime, also check `rocm examine`: it prints no +`wsl_runtime_librocdxg` line for a runtime installed by this build (the install +linked `librocdxg.so` beside the runtime's `libhsa-runtime64`), and for an older +runtime the line's `ln -sfn` command, once run, makes it disappear. Then run +`/bin/python -I -c "import torch; print(torch.zeros(1, device='cuda'))"`. + Do not set `ROCM_CLI_THEROCK_FAMILY` during normal setup tests. rocm-cli should detect the right TheRock package family or tell the user what is missing. @@ -108,9 +114,37 @@ rocm runtimes list Expected result: - `rocm examine` shows a managed runtime. -- `rocm runtimes list` shows the runtime key for the installed TheRock venv. +- `rocm runtimes list` shows the runtime key for the installed TheRock venv, + and under it a `provenance:` line naming the `channel`, the package `source`, + the `source_layout_generation` and the exact `device_target` the install + printed. A fact recorded nowhere, or recorded blank, reads `unrecorded`; + none is guessed (the target is never the family). +- `rocm examine` reports the same four facts for the active runtime as its + `active_runtime_channel`, `active_runtime_source`, + `active_runtime_source_layout_generation` and `active_runtime_device_target` + lines. - The active runtime is ready, or the output gives one clear next command. +Then read the same listing as JSON: + +```powershell +rocm runtimes list --json | python -m json.tool +``` + +Expected result: + +- `python -m json.tool` accepts the output and prints it back. Only stdout + enters the pipeline (stderr still shows on the console), so this checks that + stdout holds the document alone; a parse error means something else, such as + a banner, progress or hint line, reached stdout. +- The runtime the text form marks `*` has `"active": true`, and it is the only + one; the runtime marked `-`, if any, has `"rollback_target": true`. +- The runtime marked active is the one `rocm examine` reports: its + `install_root` is the `active_runtime_root` that `rocm examine` prints. +- Each runtime's `provenance` object names the same `channel`, `source`, + `source_layout_generation` and `device_target` as its `provenance:` line in + the text, with `null` where the text shows `unrecorded`. + Optional: an unreadable settings file. Copy `config.json` from the config directory aside, cut the original off part-way through, and run `rocm version` without `ROCM_CLI_DATA_DIR` set. Expected result: @@ -469,6 +503,33 @@ folder run `rocm engines install lemonade --yes`. Expected result: carries on: Lemonade picks the backend after this point and a smaller one may fit. (Stop it before the download if you do not want one.) +To check the refusal of a Linux or WSL host too old for the pinned Lemonade +server (no download needed): the pinned build needs glibc 2.38 and OpenSSL 3 +(`libssl.so.3`). On a real older host, such as Ubuntu 22.04 (glibc 2.35) or +Ubuntu 20.04 (glibc 2.31, OpenSSL 1.1 only), run +`rocm engines install lemonade --yes` in a fresh data folder. On a newer host, +use a build with the `rocm/e2e-test-hooks` feature and set `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and +`ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` first. Expected result: + +- It fails with `this host's libraries are too old for the Lemonade + server: it needs glibc 2.38 or newer (this host's libc.so.6 provides up to + GLIBC_)`, followed + on a host without OpenSSL 3 by `and OpenSSL 3.0.0 or newer (this host's loader + cache has no libssl.so.3)`, then `Nothing was downloaded.`, and ends by + naming `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1` as the way to skip + the check. +- The data folder's `engines/lemonade` has no `downloads` or `runtime` + subfolder, and `rocm storage report` lists no Lemonade install. +- Run it again with `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`: it prints + the same shortfall as a `warning:` ending `going ahead because + ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set.` and goes on to install + (on a real older host the server then fails to start). +- Where the pinned Lemonade is already installed, the refusal ends `Nothing was + changed: the Lemonade already installed in stays as it is, + but its server cannot start on this host either.` instead, and that folder is + unchanged. +- On Ubuntu 24.04 or newer the same command installs Lemonade as usual. + With room to spare, on Windows from an interactive console: install Lemonade with no runtime active, then `rocm install sdk`, then `rocm serve `. Expected result: @@ -1021,14 +1082,18 @@ Expected result: - After a successful removal, `rocm services list --all` no longer lists the record, `rocm services logs ` fails, and all of `/services/.json`, `/services/.log`, - `/engines//state/.json` and any - `/services/.endpoint-key` are gone. + `/engines//state/.json`, any + `/services/.endpoint-key`, and any + `/services/.endpoint-key.tmp---` (a copy + of the key an interrupted save left behind) are gone; the summary's + `files removed:` count includes that copy. - `/services/launch.lock` is untouched — it is shared by every launch. - `rocm services prune --dry-run` prints the plan and removes nothing. (It still refreshes each record against the real processes, so a record whose server has since died can have its status corrected on disk; nothing is deleted.) - `rocm services prune` leaves running servers alone and says how many it - skipped, and also removes leftover engine state files whose record is gone. + skipped, and also removes leftover engine state files, logs, endpoint keys + and interrupted-save key copies whose record is gone. - With no age argument, a record written in the last 24 hours is left alone, and the output both counts it ("too recent, kept: 1") and names the flag to include it: `rocm services prune --any-age --yes`. @@ -1111,6 +1176,35 @@ Expected result: first one's "yes" not actually exiting anything — is the bug this fixes (issue #145). +#### Endpoint key permissions + +A server's API key file must be readable by the user who started it and nobody +else. On Linux, serve with a key and check the file's mode: + +```bash +rocm serve --managed --require-api-key +stat -c '%a %U' /services/.endpoint-key +``` + +Expected result: `600` and your user name. + +On native Windows a key-protected `rocm serve` has no engine that can run +behind it: Lemonade refuses a key-protected bind there, vLLM does not run +natively, and `--engine aim` stores the key but its server then cannot +start, because rocm-cli supports AIM GPU passthrough only on Linux and WSL and +AIM serves only on a GPU. (What happens to that service's key file when the +launch fails has not been verified on a Windows host.) Check instead +`rocm remote serve `, which stores its session key with the +same owner-only file. From PowerShell, after it reports the endpoint: + +```powershell +icacls \remote-sessions\.endpoint-key +``` + +Expected result: exactly one entry, for your own account, with `(F)` and no +`(I)` (inherited) flag — no `Administrators`, `SYSTEM`, `Users` or +`Authenticated Users` entry, even when `` is a folder those can read. + ### 4a. Dashboard Doctor Report On a machine whose hardware `rocm diagnose --report` can describe (see @@ -1408,7 +1502,64 @@ rocm serve --engine vllm --gpu-memory-utilization 1 --verbose already-running summary with no memory note, whatever that service's own log contains. Read that service's log with `rocm services logs `. -## 7. ComfyUI Verification +### 6c. The tensorizer upgrade on Python 3.14 + +Needs a Linux or WSL host with a ROCm 10.1.0 runtime on Python 3.14 (the only +published vLLM build for that line). On a fresh runtime: + +```bash +rocm install sdk --channel release --version 10.1.0 --yes +rocm engines install vllm --runtime-id +``` + +The install output carries a warning that vLLM's pinned tensorizer cannot be +imported and was upgraded, then `dependency_check: expected_divergence` with a +`note:` naming the tensorizer release the runtime holds and `action: none`, not +`dependency_check: violated`. Confirm with +`/bin/python -c "import tensorizer.serialization"`, which must import +cleanly. Run the install a second time: no upgrade warning, the same +`expected_divergence` block, and vLLM is not reinstalled. `rocm serve` must then +get past engine start-up (it no longer fails with +`KeyError: 'crypto_stream_salsa20_NONCEBYTES'`). + +On some GPU families `rocm install sdk` installs vLLM itself +(`maybe_auto_install_sdk_preferred_engine`), so the same upgrade warning and the +`expected_divergence` block appear at the end of that command, a step earlier; +the `rocm engines install vllm` that follows then shows the block without the +warning, and nothing more needs doing. + +## 7. vLLM Endpoint Key Coverage + +Needs a Linux or WSL host where the vLLM engine can start. Check that an +endpoint key covers every route the vLLM server answers, not only `/v1`: + +```bash +export ROCM_SERVE_API_KEY=manual-test-key +rocm serve Qwen/Qwen3.5-0.8B --engine vllm --require-api-key --managed +PORT= +curl -s -o /dev/null -w '%{http_code}\n' -X POST "http://127.0.0.1:$PORT/invocations" \ + -H 'Content-Type: application/json' \ + -d '{"model":"Qwen/Qwen3.5-0.8B","messages":[{"role":"user","content":"hi"}]}' +curl -s -o /dev/null -w '%{http_code}\n' -X POST "http://127.0.0.1:$PORT/tokenize" \ + -H 'Content-Type: application/json' -d '{"model":"Qwen/Qwen3.5-0.8B","prompt":"hi"}' +curl -s -o /dev/null -w '%{http_code}\n' "http://127.0.0.1:$PORT/health" +``` + +Expected result: + +- The `/invocations` and `/tokenize` calls print `401`. +- Repeating them with `-H "Authorization: Bearer $ROCM_SERVE_API_KEY"` prints + `200`. +- The `/health` call prints `200` without the key. `GET`/`HEAD /health` and + CORS preflights are the only requests the server answers without it. + +Stop the service afterwards (`rocm services stop --yes`). The +automated scenario is the `@nightly` `serve-vllm-key-guards-every-route` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature), +which runs after merge in the nightly runs on Instinct GPUs (MI300X, and MI350P +on the shared GPU queue). + +## 8. ComfyUI Verification ComfyUI is managed as an app surface. It should start a local web server and show the URL to open: @@ -1487,7 +1638,7 @@ python scripts\comfyui_therock_gpu_test.py This test may download a small checkpoint and submit a cat image workflow through the ComfyUI HTTP API. -## 8. Optional Cloud Provider Key +## 9. Optional Cloud Provider Key Local ROCm use does not need a cloud provider key. If you want to test OpenAI or Anthropic provider setup, save the key through stdin so it does not land in @@ -1512,7 +1663,7 @@ To remove the saved key: rocm config clear-provider-key openai ``` -## 9. Optional Provider-Assisted Planning +## 10. Optional Provider-Assisted Planning Most users should leave this off. To test ambiguity resolution with an already running local provider service: @@ -1540,7 +1691,7 @@ To turn provider-assisted planning back off: rocm config clear-planner-provider ``` -## 10. Uninstall +## 11. Uninstall Run this last, on a machine installed with the release installer (not a `cargo` build, whose binaries uninstall leaves alone): @@ -1616,7 +1767,7 @@ Expected result: either case. The same prompt is shared by `rocm fix` and the SDK-overwrite prompt, so the same answers behave the same way there. -## 11. AIM (Docker) Engine Verification +## 12. AIM (Docker) Engine Verification Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must succeed). The catalog entry used below has profiles for Radeon PRO W7900 and @@ -1681,7 +1832,7 @@ Expected result: - On native Windows, the same command fails explicitly instead of falling back to another engine. -## 11. Background Automation Helper +## 12. Background Automation Helper The helper (`rocm daemon`) runs only while an automation watcher is enabled. Start from a fresh config (all watchers off) and use a scratch @@ -1712,7 +1863,7 @@ Expected result: seconds before the helper exits) still ends with a helper running: enable waits for an exiting helper to finish, then starts a new one. -## 12. Context-Size Estimate for a GGUF Model +## 13. Context-Size Estimate for a GGUF Model Needs a host with a Radeon GPU (a discrete card: an APU answers `undetermined` by design) and network access to Hugging Face. The point is to check the @@ -1775,7 +1926,7 @@ Finally, `rocm diagnose --report --model /:Q4_K_M --context-size 8192` prints the report with no model section, and stderr says that `--model, --context-size were ignored` and nothing about the model was fetched. -## 13. Pasting An Error Into `rocm diagnose` +## 14. Pasting An Error Into `rocm diagnose` `rocm diagnose --symptom-file` takes a pasted error without shell quoting. Copy a line that would break an argument, for example: @@ -1813,7 +1964,7 @@ Expected result: paste with a non-ASCII character in it (`é`, `→`, a typographic quote) reports the same causes as in PowerShell 7. -## 14. Which Causes a Pasted Error Contributed To +## 15. Which Causes a Pasted Error Contributed To `rocm diagnose --symptom` scores the catalog from the examined host's state and the pasted text together, so a finding about the host can outrank the entry the @@ -1841,7 +1992,7 @@ HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --json The e2e scenarios `diagnose-62`, `diagnose-63` and `diagnose-64` run these three checks on every lane. -## 15. Dashboard Chat With A Small-Context Local Model +## 16. Dashboard Chat With A Small-Context Local Model Needs a local OpenAI-compatible endpoint serving a model with a small context — llama.cpp's default `n_ctx` is 32768, which is the case this checks. The @@ -1882,3 +2033,107 @@ Expected result: read (N of M tokens). Try a more specific question, or a model with a larger context.` — and `rocm-cli.log` should still carry the provider's own body with `exceed_context_size_error` in it. + +## 17. Serve Smoke Test Failure + +Needs any engine `rocm serve` can start (the behaviour is not vLLM-specific) and +an interactive terminal: the smoke test and its summary run only there. The +behaviour under test is described in +[the README](../README.md#model-serving); the automated coverage is listed in +[testing.md](testing.md#serve-smoke-test-failure). + +The repeatable route is the E2E scenario, which needs no GPU but runs on Linux +and WSL only (`@requires-os:linux`): + +```bash +cargo xtask e2e -- -n "serve-60" +``` + +(`@id:serve-smoke-failure-summary`.) + +By hand, a failing first chat request is hard to provoke (the engine has to pass +its readiness probe and then fail or die on the first chat request), so check the +points below when you see it, for example after an engine is killed right as it +becomes ready: + +- The heading reads `Deployment summary (smoke test failed)`, not + `Deployment summary`. +- The `status` row reads `started, smoke test failed`, not `ready`. +- `time to first token` and `throughput (approx)` read `n/a`, and a `note:` line + gives a one-line reason and names `rocm logs --service `. +- The command exits 0 and ROCm CLI does not stop the server. If the engine died + there is no server left: `rocm logs --service ` still shows its output, and + the next `rocm services list` finds the engine gone and records the service as + `stopped` (or `failed` for AIM), so it drops out of the default list and shows only under + `rocm services list --all`. If the engine is still up and only the chat request + failed, the service stays listed and its recorded status stays `ready`. +- `rocm serve --no-smoke-test` prints the plain `Deployment summary` with + no smoke-test note, and re-running against an already-running service prints + the `(already running)` summary with no smoke-test note. +- Some failures are deliberately not detected and still read as `ready` (an error + sent as a `data:` payload inside a `200` stream, a `200` answer with no `data:` + line, a close-delimited stream cut after a token); the full list is in + [testing.md](testing.md#serve-smoke-test-failure). +- A reasoning model that spends its small token budget before emitting any + content is not a failure: the summary stays `ready` with the two metric rows + `n/a` and no smoke-test note. + +## 18. `rocm fix` When A Step Does Not Finish Or A Read Fails + +The automated scenarios (`diagnose-15`, `diagnose-68` on bare-metal Linux; +`diagnose-39`, `diagnose-69`, `diagnose-70` on Windows) cover these; repeat +them by hand on a release build. + +Linux, a helper that never finishes. Stand a hanging `sudo` in for the real +one, on a `PATH` that holds nothing else (so the run is not as root and changes +no group): + +```bash +ROCM=$(command -v rocm) +FAKE=$(mktemp -d) +printf '#!/bin/sh\nexec /bin/sleep 600\n' > "$FAKE/sudo" +printf '#!/bin/sh\nexit 1\n' > "$FAKE/usermod" +chmod +x "$FAKE/sudo" "$FAKE/usermod" +PATH="$FAKE" "$ROCM" fix fix-4-render-group --yes; echo "exit $?" +``` + +Expected result: + +- After about a minute, stderr says ``sudo usermod timed out after 60s and was + stopped; group membership may or may not have changed. Check with + `id -nG `, then re-run `rocm fix fix-4-render-group` if it did not.`` +- The exit code is 4. Nothing says `NOT changed`, and there is no + `Added to render,video.` line. +- `pgrep -f 'sleep 600'` finds nothing afterwards. + +Windows, a read that fails. In a PowerShell window, empty `PATH` for that window +only and start `rocm.exe` by its full path: + +```powershell +$rocm = (Get-Command rocm).Source +$env:Path = '' +& $rocm fix fix-2-unset-override --yes; "exit $LASTEXITCODE" +& $rocm fix fix-9-igpu-dgpu --device-index 1 --dry-run; "exit $LASTEXITCODE" +& $rocm fix fix-2-unset-override --dry-run; "exit $LASTEXITCODE" +& $rocm fix fix-6-path --dry-run; "exit $LASTEXITCODE" +``` + +Expected result: + +- fix-2 prints `Could not read the current User HSA_OVERRIDE_GFX_VERSION: + PowerShell could not be started: it was not found; nothing was changed.` on + stderr and exits 4. It does not print `No persistent HSA_OVERRIDE_GFX_VERSION + found` or `You're done`. +- fix-9, a dry run here, prints the same line for `HIP_VISIBLE_DEVICES`, exits + 4, and prints no `Plan: persist HIP_VISIBLE_DEVICES` line (the catalog entry + that starts `Fix:` is still printed before it). It is run as a dry + run because the empty `PATH` hides PowerShell but not `setx.exe`: a fix-9 that + wrongly went on would set your real User-scope value. +- The fix-2 dry run stops the same way: exit 4 and the read-failure line, after + the catalog entry, with no `The CLI runs (arguments, not a line to paste)` + line. +- fix-6 (needs a HIP SDK install; without one it exits 3 before reading) + prints `Could not read the current User PATH: PowerShell + could not be started: it was not found; nothing was changed.`, exits 4, and + prints no `Plan: prepend` line. +- Close the window afterwards; the empty `PATH` was only that window's. diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index c8a3bdbb63..380ba4de2d 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -28,10 +28,10 @@ The lane set is whatever GPU models the queue has. The 11-13 run the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no longer has a W7900 lane. -**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so -the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` -and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing -here replaces them. Which checks are required before merging is a repository +**The remaining gap is Strix Halo.** The queue has no Strix lane and runs Linux +pods, so the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool, +in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); +nothing here replaces them. Which checks are required before merging is a repository setting, outside this page. This page is the **contract** between this repository and the queue side: what @@ -56,6 +56,69 @@ third dispatch, or the nightly schedule, replaces a dispatch still waiting and that one never runs. The branch must exist in this repository; otherwise the job fails at once. +## Pull requests + +A pull request from this repository also runs its MI300X check on the queue. +The gate job `e2e-gpu-queue` in +[`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml) runs on a +GitHub-hosted runner, holds no queue credential and needs only `actions: write`: +it dispatches [`queue-pr-run.yml`](../.github/workflows/queue-pr-run.yml) on +`main` with the pull request's branch, waits for that run, and publishes the +release-channel report of the MI300X lane as `e2e-gpu-report`, the MI300X cell +of the consolidated report. The run's files are the artifact +`queue-pr-run-results` from `queue-pr-run.yml`, as `queue-run-results` is +`queue-run.yml`'s. +The self-hosted MI300X lane of the same pull request uploads +`e2e-gpu-selfhosted-report`, shown beside it as "MI300X (self-hosted)": the queue +run is the primary MI300X check and the self-hosted one the second. + +A pull request's run is a preset of the queue side: one GPU model (MI300X), the +`release` channel alone and no `@nightly` scenarios, a lane that waits at most 45 +minutes to be started and runs at most 75, and a limited number of such runs +at once (the queue side sets the limit). The +queue side does not take one while the nightly goes, or when it has no healthy +MI300X worker: the gate job then fails with the reason the queue gave, which is +capacity, not a failing test (the consolidated report's missing-report check +then flags the lane too). The nightly window is the exception, being by design: +the gate ends green with a note in its summary to re-run once the window has +passed (the queue side defines the window), and +the consolidated report reads the lane as skipped, not missing. A superseded push cancels the gate job, which cancels +the run it started, freeing the runner; the queue side's lanes are left to their +own budgets. The queue run tests the branch as it is when the run starts, not +the pull request's merge with its base, which the self-hosted lane checks out. +A pull request from a fork gets no queue run, since +its token cannot dispatch, and neither does one from Dependabot, whose token is +read-only whatever the workflow asks for (its gate is skipped) and the queue only checks out branches of this +repository. `e2e-selfhosted.yml` is the only workflow that may name +`queue-pr-run.yml`; the runner is set up to run `queue-pr-run.yml` only as it is +on `main`, never the pull request's own copy of any workflow. + +**Merge order.** The queue side (the runner's allow-list and the pull request +run kind) goes live before the gate does anything. The gate is switched by the +repository variable `QUEUE_PR_GATE`: until it is `true` the gate ends green with +a note in its summary, as a skip, not a pass of the MI300X check, and the +self-hosted `e2e-gpu` is the only MI300X check that runs. Set it once the queue +side is deployed. Without it, merging `queue-pr-run.yml` first would turn every +pull request red. + +The gate and the queue side agree on two things, and the queue side is not in +this repository, so the gate's tests pin the gate's half only (the +`queue_gate_script_*` tests in +[workflow_contract.rs](../xtask/src/workflow_contract.rs), whose fixtures use +the same two constants the script is checked against): + +- **A refusal.** A run the queue did not take publishes `refused.txt` in + `queue-pr-run-results`, whose first line is the reason. A reason that starts + with `nightly:` is the nightly window and skips the gate; any other reason + fails it. +- **The report.** The MI300X lane's release-channel report is + `mi300x/release/results/report.json` in that artifact, and its whole + `results` directory is the job's report. A run with none fails the gate. + +GitHub cancels an older run of a branch that is still pending when a newer run +of the same branch is dispatched; the gate of the older run then sees a run +that left no report, and fails with that message. + ## Publishing a finished run Dispatch `nightly-queue.yml` with a run's `request_id` — the id its @@ -125,7 +188,11 @@ limit (see the run's inputs above). Each channel starts with the GPU preflight, channel's job in `nightly.yml` does, so a model server the previous channel's suite left on the GPU is reclaimed first; the lane log shows it. Both always run, so a failing channel does not hide the other, and the lane fails if either -did. +did. The default command also declares that every lane has a GPU +(`E2E_EXPECT_GPU`, default `1`), so a channel whose suite finds no usable AMD +GPU, or runs no GPU scenario (`@requires-gpu` or `@requires-multi-gpu`), fails +instead of passing with every GPU scenario skipped; see "Lane verdict" in the +[E2E README](../tests/e2e-cucumber/README.md). A lane starts cold, so its first steps fetch from the network: the build packages, the rustup installer, the toolchain and the crates for the host @@ -135,10 +202,15 @@ model downloads are left to the tools doing them, so a model that downloads more slowly than its scenario waits fails that scenario. The lane fetches only the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. -The queue's results are an additional signal, not a replacement: the lanes -write only into the queue artifact, and the `e2e-gpu--report` -artifacts that `nightly.yml`'s consolidated report reads are not produced from -them. +The queue is the only scheduled run of the R9700, MI350P and W7900 lanes: their +self-hosted nightly jobs were removed from `nightly.yml`. The MI300X nightly +job is still there, and the queue's `mi300x` lane is an additional signal to it. +The lanes write only into the queue artifact, and the +`e2e-gpu--report` artifacts that `nightly.yml`'s consolidated report +reads are not produced from them. The report fetches the queue's artifact +itself instead, so the support matrix, the `E2E-Support-Matrix` and +`Release-Readiness` wiki pages and the release-readiness verdict keep every +GPU the queue tests: see [The support matrix and the wiki](#the-support-matrix-and-the-wiki). ## Output @@ -192,12 +264,85 @@ that line is what counts: a blocking lane without a result fails the run. result is not a pass, but they point at queue capacity or infrastructure rather than at a regression. +## The support matrix and the wiki + +`nightly.yml`'s `e2e-report-nightly` job builds the consolidated support +matrix, which `publish-e2e-wiki` publishes as the `E2E-Support-Matrix` wiki page +and `release-readiness-nightly` reads its GO/NO-GO verdict from (see +[ci-hardware-testing.md](ci-hardware-testing.md)). It also reads the queue's +lanes, from the queue's own results, so retiring a self-hosted nightly lane +that the queue covers drops no GPU from either page, and a GPU that only the +queue has (MI325X, MI350X, MI355X) is on them. + +- **Which run.** The newest `queue-run-results` artifact of a run on this + repository's `main` that finished, and started, in the last 24 hours, + whatever the run's conclusion was: a red lane fails the run, and its artifact + is still uploaded. The job finds the artifact by name, not through the + workflow that makes it, because no other workflow may name that one (see + below). The run also has to have tested `main`: a run started from `main` can + be asked to test another branch (the `branch` input), and the job log names + the branch it did. `queue-run.yml` starts at 02:17 UTC and the nightly + starts at 06:00 UTC, its report job after its GPU lanes have finished, so a + run that took under about three and a half hours is in the night's report. A + run that is still going, that started on an earlier day or that tested + another branch is not imported, and neither is a run published again by + `nightly-queue.yml`. The job looks at the five newest runs of the window and + takes the first that tested `main`, so a run of another branch does not hide + an earlier run of `main`. +- **Which lanes, and whether each blocks.** The artifact holds the lanes' + files and nothing else about the run, so the job reads the run's job log, whose + lane matrix is the only place the queue prints which lanes it ran and which of + them can fail the run (the `GATE` column). Only the tested branch and commit + and each lane's name, phase, exit status and gate are taken from it, each + checked against a strict pattern first. A lane whose name is not a plain + lowercase word (letters, digits and dashes, and not ending in `-advisory`, + `-release` or `-nightly`) is shown as `unnamed-1` and so on, as a blocking + lane with no result. Without the log nothing is imported. +- **What is imported.** For each lane and channel, + `//results/{report.json,platform.json,commands.jsonl}`, the + three files the report reads, as an artifact named + `e2e-queue-[-advisory]--report`. The `lane.log`, the JUnit and + HTML output and the harness diagnostics are not copied, and nothing the + queue's scan withheld is brought back. The lane's `platform_slug` is replaced + by `queue--`: the slug is the report's column identity, and + queue lanes of different models report the same one (an MI325X lane probes as + `mi300x`, an MI350X or MI355X lane as `mi350p`). The lane command therefore + has to keep writing those files to `/out//results`, as + [`scripts/run-e2e-lane.sh`](../scripts/run-e2e-lane.sh) does. +- **How it reads.** A queue row is labelled `MI325X (queue) Linux (release)`, + so it is not confused with the self-hosted MI300X lane, and a lane the queue + does not block on is `MI325X (queue, advisory) Linux (release)`. The page + opens with a `Queue lanes` table: one row per lane the queue ran, with its + gate, phase, exit status and the channels that reported. +- **What counts towards the verdict.** The same as for the queue's own run. A + blocking lane's failures are in the `E2E` failure count. An advisory lane's are + shown but left out of it. A lane that came back with no result, or whose + channel is missing after it did not finish cleanly, is shown as `EMPTY` and + counts for a blocking lane, because a blocking lane without a result is not + a pass. +- **When there is nothing to import.** No run in the window, a run of another + branch, an artifact or log that cannot be downloaded or an import that fails + never fails the nightly report: the page opens with a `Queue lanes` note + saying that no queue lanes are in it tonight and why, and the rest is built as + before. A queue that never ran is not a pass, so the + release-readiness verdict is `NO-GO` with the failing gate `queue lanes + absent: `; its `E2E` row is marked "queue lanes missing" and a warning + under the table names the reason. + +What is public follows from the queue's own rules: the artifact is public and +already scanned, only three of its files are used, and the log is the public job +log. + ## What the workflow requires -- a self-hosted Linux runner registered with the `queue-report` label alone, +- self-hosted Linux runners registered with the `queue-report` label alone, without GitHub's default `self-hosted`, OS and architecture labels, so only a - job naming `queue-report` lands on it, providing - the queue side's publish command and read access to the queue side's results + job naming `queue-report` lands on them, each providing + the queue side's publish command and read access to the queue side's results. + Each run holds a runner for as long as it lasts, so there are as many runners + as runs may go at once (the queue side limits pull request runs to that + number); a job beyond them waits in GitHub's queue, outside the queue side's + limits, and a gate job that waits for such a job can reach its own timeout - a registered runner name and host name that do not identify internal infrastructure, since GitHub prints both into every job log @@ -211,7 +356,7 @@ this repository can still name the runner's label, which is one reason the runner holds no credential, below. On `main`, the tests also refuse any other workflow that names the `queue-report` label or these workflows, so none sends a job to the runner or waits on them by mistake. The runner is set up to -run only these two workflows as they are on `main`, by a check on the runner +run only the allow-listed queue workflows as they are on `main`, by a check on the runner that runs before any step. That check lives outside this repository, so nothing here can verify it; the runner is still treated as untrusted, as above and below. Whether this @@ -251,7 +396,7 @@ file. credential and sends nothing to the queue: the runner, which sits next to the queue side, asks it for a run of a branch, and the queue side starts it with its own credential. It believes the request because of where it comes from, -which is why the runner is set up to run only these two workflows from `main` +which is why the runner is set up to run only the allow-listed queue workflows from `main` (a setting outside this repository). A request can start a run of a branch already in this repository, and that branch's lane command then runs in the lane pods. The lane spec gives those pods no queue or cluster credential and diff --git a/docs/release-trust.md b/docs/release-trust.md index afddf40ba2..d12575da33 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -11,7 +11,8 @@ rocm-cli release packages are protected in two layers: 1. Every bundle has a `.sha256` checksum, and installers verify it before activation. 2. Release packaging can emit detached RSA/SHA-256 `.sig` files, and release - and nightly CI now require those signatures for published assets. + and nightly CI now require those signatures for every bundle archive they + publish, the archives the installers download. ## Signing Inputs @@ -28,9 +29,14 @@ ROCM_CLI_SIGNING_PRIVATE_KEY_PATH ``` Stable release and nightly CI set `ROCM_CLI_REQUIRE_SIGNATURE=1`, so packaging -fails if no signing key is configured or if the `.sig` file is not produced. -Upload steps also fail if signature assets are missing instead of publishing an -unsigned release or nightly. +fails if no signing key is configured or if the `.sig` file is not produced. The +packaging steps also stop if the `.sig` is missing and run the dist verifier +with `--require-signatures` before anything is uploaded, and the nightly publish +job verifies the combined asset set the same way before replacing the public +`nightly` release, so neither workflow publishes an unsigned bundle archive. The +nightly release's convenience assets, attached after those checks, carry no +sidecars (see +[Remaining Owner Step](#remaining-owner-step)). Release CI also runs the cross-platform dist verifier before upload. Release jobs name both the versioned archive and the generic installer-facing archive, @@ -113,8 +119,20 @@ they are not trust roots. ## Installer Verification -Installers always verify checksums. Signature verification runs when a public -key is supplied or `ROCM_CLI_REQUIRE_SIGNATURE=1` is set. +Installers always verify checksums, and while a release key is pinned they +verify the detached signature on every install whatever the channel. The +release public key is pinned in both [install.sh](../install.sh) and +[install.ps1](../install.ps1) (canonical copy +[rocm-cli-release-current-public.pem](keys/rocm-cli-release-current-public.pem)), +so a key always resolves and a missing `.sig` sidecar fails the install. An +unset or `0` `ROCM_CLI_REQUIRE_SIGNATURE` does not turn verification off. A key +supplied through `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH` or +`ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` (or `install.ps1`'s `-SigningPublicKeyPath` or +`-SigningPublicKeyPem`) replaces the pinned keys rather than adding to them, for +example for a private mirror signed with its own key. Were both pinned slots +emptied, an install with no key supplied would skip verification, unless +`ROCM_CLI_REQUIRE_SIGNATURE=1` (or `install.ps1`'s `-RequireSignature`) is set, +which makes it fail for want of a key. The detached `.sig` sidecar is the raw RSASSA-PKCS#1 v1.5 signature over the archive's SHA-256 digest, verified against the SubjectPublicKeyInfo @@ -128,10 +146,11 @@ on both Windows PowerShell 5.1 (.NET Framework 4.6+, which the CMS/Authenticode blobs, not a raw detached signature checked against a bare public key. +To verify against a different key: + Linux: ```bash -ROCM_CLI_REQUIRE_SIGNATURE=1 \ ROCM_CLI_SIGNING_PUBLIC_KEY_PATH=/path/to/rocm-cli-signing-public-key.pem \ sh ./install.sh release ``` @@ -139,16 +158,19 @@ sh ./install.sh release Windows: ```powershell -$env:ROCM_CLI_REQUIRE_SIGNATURE = "1" .\install.ps1 release -SigningPublicKeyPath C:\path\to\rocm-cli-signing-public-key.pem ``` The `@lifecycle` install-lifecycle E2E scenarios cover public-key path and PEM installer inputs, bad checksums, bad detached signatures, missing required -`.sig` sidecars, and required-signature mode without a public key. The Windows -scenarios additionally run a full signed install over a loopback HTTP download, -proving `install.ps1` verifies with native .NET crypto. All of those checks -happen before activation. Run the current host's set with +`.sig` sidecars, and an install with no public key supplied, which falls back to +the pinned release key and rejects the locally signed archive. On Linux two more +run without asking for a signature: with `ROCM_CLI_REQUIRE_SIGNATURE=0`, the +pinned key still rejects the locally signed archive when a specific version is +installed, and with the variable unset a missing `.sig` still fails a release +install. The Windows scenarios additionally run a full signed install over a +loopback HTTP download, proving `install.ps1` verifies with native .NET crypto. +All of those checks happen before activation. Run the current host's set with `E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e` (see `docs/testing.md`). @@ -269,6 +291,13 @@ Set the gate and the specific base variable together, and only in tests or deliberate manual QA against a fixture server. The `therock-next` E2E scenarios use exactly this pairing to exercise next-layout dispatch hermetically. +A wheel install records the pip index it resolved from, including an overridden +base, in the runtime's manifest as `index_url`. `rocm engines install vllm` reads +that value back for a ROCm 10.2 runtime and fetches the SDK's own torch stack from +it, without checking its scheme or host, so an override used for one install keeps +being used for that runtime's later vLLM installs, after the override is unset. +Remove and reinstall the runtime, or edit its manifest, to stop that. + ## ComfyUI Source Archive Override rocm-cli hardcodes the URL it downloads the ComfyUI source archive from. It @@ -377,31 +406,52 @@ node, as it always has, so the veto does not apply there. ## Remaining Owner Step -The repo still needs a real project-owned public signing key and matching -private release secret, plus the public metadata and model recipe index keys, -signed sidecar publication, and hosted production recipe/source-policy -documents. Do not generate those production keys inside CI. Once published, the -installers, metadata cache, and model recipe index can make release-channel -signature verification mandatory by default with those keys as trust roots. - -Those trust roots are pinned copies of the published keys: the -`PINNED_RELEASE_PUBLIC_KEY_CURRENT`/`_NEXT` slots in -[install.sh](../install.sh), `$PinnedReleasePublicKeyCurrent`/`Next` in -[install.ps1](../install.ps1), and -`PINNED_METADATA_PUBLIC_KEY_PEM` in -[fetch.rs](../apps/rocm/src/therock/fetch.rs). `cargo xtask verify-pinned-keys` -fails when a pinned copy differs from its canonical file under `docs/keys/`. The -release and nightly workflows always run it; pull-request CI runs it when a +The release signing key is published as +[rocm-cli-release-current-public.pem](keys/rocm-cli-release-current-public.pem) +and pinned in both installers. The release and nightly workflows sign every +archive the installers download with the matching private key and verify the +signature before upload, and the installers already enforce it (see +[Installer Verification](#installer-verification)). The nightly release also +carries convenience assets uploaded with neither a `.sig` nor a `.sha256` of +their own: `linux-binaries.tar.gz` and `windows-binaries.zip`, byte-identical +copies of the signed Linux and Windows nightly archives, and the standalone +`rocm` and `rocm.exe` binaries, extracted from those archives. No installer +downloads them; to check what you download, prefer the signed archives. + +Two owner-controlled inputs are still not published: + +- The runtime metadata signing key and the `.sig` sidecars for the metadata. + Until they are, `PINNED_METADATA_PUBLIC_KEY_PEM` stays `""` and metadata + signature verification stays opt-in through the + `ROCM_CLI_METADATA_PUBLIC_KEY_PATH`/`_PEM` env vars. +- The model recipe index signing key, plus a hosted signed production recipe + index and source-policy documents. The CLI pins no key for the index and reads + one only from `ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH`. + +Do not generate those production keys inside CI. + +The trust-root slots are the `PINNED_RELEASE_PUBLIC_KEY_CURRENT`/`_NEXT` slots +in [install.sh](../install.sh), `$PinnedReleasePublicKeyCurrent`/`Next` in +[install.ps1](../install.ps1), and `PINNED_METADATA_PUBLIC_KEY_PEM` in +[fetch.rs](../apps/rocm/src/therock/fetch.rs). A slot holds a pinned copy of a +published key, and a slot whose key is not published yet (the `_NEXT`/`Next` +release slots and the metadata slot) must stay `""`, which +`cargo xtask verify-pinned-keys` enforces. It also fails when a pinned copy +differs from its canonical file under `docs/keys/` and, where CI configures a +signing public key (`ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, else +`ROCM_CLI_SIGNING_PUBLIC_KEY_PEM`), when that key differs from the canonical +current release key, so CI verifies against the key users pin. The release and +nightly workflows always run it; pull-request CI runs it when a change matches the `heavy` path filter in [ci.yml](../.github/workflows/ci.yml), which covers `docs/keys/`, the installers, every Rust source and `xtask/`. It reads the one assignment the installer actually uses, so each slot must be assigned exactly once as a -plain declaration, a slot whose key is not published yet must stay `""`, and -`install.sh` must mark both of its slots `readonly`. The full rules, and what a -textual check cannot catch, are in +plain declaration, and `install.sh` must mark both of its slots `readonly`. +The full rules, and what a textual check cannot catch, are in [verify_pinned_keys.rs](../xtask/src/verify_pinned_keys.rs). -When those owner-controlled inputs exist, release operators can add +Once the runtime metadata public key and the signed model recipe index (with +its public key) are published, release operators can add `--require-production-trust` or set `ROCM_CLI_REQUIRE_PRODUCTION_TRUST=1` so the -dist verifier fails unless the release signing public key, runtime metadata -public key, and signed model recipe index inputs are explicitly configured. +dist verifier fails unless both are explicitly configured. The flag also checks +the release signing public key, which release and nightly CI already configure. diff --git a/docs/rocm-docs/conf.py b/docs/rocm-docs/conf.py index 01b4df6b05..0e52f7ab7a 100644 --- a/docs/rocm-docs/conf.py +++ b/docs/rocm-docs/conf.py @@ -86,7 +86,7 @@ def filter(self, record: logging.LogRecord) -> bool: ROCM_CORE_SDK_URL = "https://rocm.docs.amd.com/en/latest/" -def _point_core_sdk_link_to_latest(app, pagename, templatename, context, doctree): +def _point_core_sdk_link_to_latest(app, pagename, templatename, context, doctree): # noqa: ARG001 # On docs-X.Y.Z branches rocm-docs-core resolves every sister project to # that same version, but the ROCm Core SDK has no docs-0.3.0 build. context["projects"] = {**context["projects"], "rocm": ROCM_CORE_SDK_URL} diff --git a/docs/testing.md b/docs/testing.md index a21272ebc1..6bfa22f142 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -335,6 +335,77 @@ stderr. `an_unparseable_answer_from_a_failing_child_names_the_exit_status` pins the three cases against the fake, and `engine-ipc-02` below shows the same message in a `stop_server` report next to what the stop did. +That error also says what is wrong with what the child printed, because "failed +to parse" alone sends the reader to the engine's source: it printed nothing on +stdout, or other output before or after its answer (a log line on the wrong +stream), or JSON that is not an envelope, or something that is not JSON. The +first says only that nothing was printed. The second quotes the other output, +and the last two the start of stdout (one line, bounded, control characters +escaped); those three also state the rule an engine breaks: only its JSON +answer goes to stdout, its logs go to stderr. +Nothing is salvaged from such output. Only a top-level JSON object counts as the +answer, so a pretty-printed answer over many lines is still one answer, and an +envelope inside an array or another object is not mistaken for one. +`an_unreadable_stdout_is_described_by_what_is_wrong_with_it` covers the +classification on every platform, `an_answer_past_the_search_bound_is_not_looked_for` +the bound on how much of stdout the search reads, and +`output_besides_the_answer_on_stdout_is_refused_and_named` runs the same cases +through the fake. + +The engine protocol version (see the contract in +[engine-plugins.md](engine-plugins.md#the-stdio-protocol-contract)) is tested on +both sides of the conversation and in the rule itself. The rule is a table in +[`protocol_version.rs`](../crates/rocm-engine-protocol/src/protocol_version.rs) +(`the_caret_rule_from_this_builds_version`, every platform), and +`protocol_properties` adds that the field round-trips in both envelopes, that a +missing version is judged as `0.1.0`, and that the rule is reflexive and +symmetric. The client refuses an answer in an incompatible protocol before it +reads `ok` (`an_answer_in_an_incompatible_protocol_is_refused_whatever_it_says`). +The check before an `install` or `launch` is a state matrix against the fake, each +row asserting the error together with which requests the fake recorded: +`a_compatible_engine_is_checked_and_then_sent_the_request` (same, newer +additive and unversioned capabilities answers: two requests, the capabilities +check first), `a_handshaked_request_is_not_sent_when_the_check_fails` +(incompatible, incompatible refusal, unreadable version, an error answer, a crash +with no answer, empty stdout, contaminated stdout, not JSON: only the +capabilities request ever reached the engine), and +`a_request_without_a_handshake_is_sent_once` (every other method is +sent once, and its answer's version is what is checked). The check's own child +is pinned too: `the_check_runs_on_the_requests_environment_without_its_progress` +reads the environment the fake recorded for each run, and +`the_check_withholds_an_inherited_key_file_as_the_request_does` the command it +builds. On the engine side, `answer_engine_request` in `protocol_version.rs` +judges a request's version before its method +(`a_request_is_judged_by_its_version_before_its_method`), and one shared check, +`engine_contract::assert_engine_holds_protocol_contract` (behind the +`test-support` feature), holds every engine to the same list: an incompatible or +unreadable version refused with `incompatible_protocol` for every method in +`EngineMethod::ALL` on the stdio path and the decoded path, and for an unknown +method on the stdio path, `unknown_method` and `invalid_request` for the other +unusable requests, and this build's version on every answer. Each first-party engine runs it +(`stdio_answers_hold_the_engine_protocol_contract` in its `lib.rs`), and +`every_built_in_engine_holds_the_protocol_contract` in `apps/rocm` runs it over +every built-in engine through `answer_builtin_engine_stdio`, the function +`rocm __engine-stdio` answers with; the verb's own stdin reading is covered by +e2e `engine-ipc-08` to `-12`, which run `rocm __engine-stdio vllm`, for `vllm` +only; of those, `-09` to `-12` fail when the verb bypasses the shared reader. +Each engine's `tests/stdio_binary.rs` runs its built `rocm-engine- +stdio` binary and checks that a request for another protocol and input that is +not JSON are each answered with an error envelope and exit status 0. +The shared check is itself tested in both directions: an engine that holds the +contract passes it, and one that ignores the version, or reads it only on the +decoded path, fails it. `the_contract_check_fails_an_engine_that_breaks_any_one_rule` +adds six engines that each break one rule (named in its doc comment), and the +check fails each of them; the check's other rules have no such engine. `engine-ipc-03` to `-07` below show the daemon's side and +`engine-ipc-08` to `-12` the engine's. + +```bash +cargo test -p rocm-engine-protocol +cargo test -p rocm-engine-lemonade -p rocm-engine-vllm -p rocm-engine-aim --lib protocol_contract +cargo test -p rocm-engine-lemonade -p rocm-engine-vllm -p rocm-engine-aim --test stdio_binary +cargo test -p rocm every_built_in_engine_holds_the_protocol_contract +``` + The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at its default action, which is how `rocm` runs, because the test harness leaves it ignored and changing it process-wide would race sibling tests. A write that is @@ -951,6 +1022,45 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3 --repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community" ``` +## E2E runs that test nothing fail + +`cargo xtask e2e` fails a run whose result could not mean anything, instead of +reporting it green: + +- before any scenario runs, when the capability probe could not read the binary + under test: `rocm examine` or `rocm engines list` could not be started (check + `ROCM_CLI_BINARY`), or could not be waited on, exited non-zero, printed + nothing, was still running after `E2E_PROBE_TIMEOUT_SECS` (whole seconds ≥ 1; + anything else uses 120), or exited but left its output open. The post-run + version probe (`collect_versions`) uses the same deadline but fails quietly, + only blanking the version columns, and does not refuse the run; +- after the run, when no scenario executed, which includes a + `cargo xtask e2e -- -n ` whose regex matches no scenario name. The + reason is written into `platform.json`, so `cargo xtask e2e-report` shows that + lane as FAIL with the reason under "Needs attention: platforms", in both its + markdown summary and its HTML report. + +`E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also +fails before any scenario if the probe found no usable AMD GPU, and after the +run if no `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU +scenarios the run's own configuration leaves out (`@nightly`, `@merge-queue`, +or anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved +but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and no +GPU scenario selected fails by design. With `0`, it only notes that GPU +scenarios are not applicable (and that the declaration is stale if a GPU is +found). Unset, as on a local run, it makes no GPU claim. Every self-hosted CI +lane sets it; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md) +and the Platforms table in [ci-hardware-testing.md](ci-hardware-testing.md). The +checks are unit-tested in [lane_verdict.rs](../tests/e2e-cucumber/src/lane_verdict.rs) +and, for each way a probe can fail, in +[capability.rs](../tests/e2e-cucumber/src/capability.rs): + +```bash +cargo test -p e2e-cucumber --lib lane_verdict +cargo test -p e2e-cucumber --lib capability::tests +cargo test -p xtask every_self_hosted_lane_declares_whether_it_has_a_gpu +``` + ## CI test selection On pull requests, CI does not test the whole workspace. The `test` job runs @@ -1671,9 +1781,9 @@ the tagging and that they stay reachable when the opt-in is set. Both scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"). -`serve-27`'s known failure on the Strix Halo pool is an xfail row in -`expectations.toml`, tracked in rocm-cli#1058; a pass there is an XPASS, which -means the row is stale. +`serve-27` failed to confirm the AIM stop on the Strix Halo pool (rocm-cli#1058) +and passed on the 2026-11-15 and 2026-11-16 nightlies, so it runs expect-pass +there with no row in `expectations.toml`. ### Lemonade managed-serve failure evidence @@ -1786,6 +1896,171 @@ List registered managed or read-only runtimes and their exact side-by-side keys: ```bash rocm runtimes list +rocm runtimes list --json +``` + +`--json` prints the same listing as one JSON document on stdout and nothing +else: unset values are `null` rather than ``, each runtime carries +`active` and `rollback_target` flags computed from the same values the text's +`*`/`-` markers are drawn from, and the empty-registry next-step hint is +text-only. The rules every `--json` command follows, and which commands have +one, are in [json-output.md](json-output.md). Scenarios +`@id:runtime-lifecycle-list-json-marks-active-and-rollback` +(`runtime-lifecycle-23`) and `@id:runtime-lifecycle-list-json-empty-registry` +(`runtime-lifecycle-24`) parse the whole of stdout as one document, so a stray +line before or after it fails them; `runtime-lifecycle-23` also checks each +runtime's values as planted, lists the same state as text and checks that the +`*`/`-` rows name the runtimes the JSON flags, and `runtime-lifecycle-07` covers +the text markers on their own. +`@id:runtime-lifecycle-list-json-reports-unusable-runtime` +(`runtime-lifecycle-25`) deletes a runtime's folder and reads +`"status": "unusable"` with the reason `install root is missing: `, and +`@id:runtime-lifecycle-list-json-reports-missing-active-manifest` +(`runtime-lifecycle-26`) deletes the active runtime's registry entry and reads +`active_problem` of kind `missing_manifest_for_key`. +`@id:runtime-lifecycle-list-json-failure-prints-nothing-on-stdout` +(`runtime-lifecycle-30`) covers the failure half of those rules: it replaces a +registry entry with a folder of the same name, so the listing cannot read it +(an entry that reads but does not parse is skipped, not an error), and checks +that `--json` exits 1 with nothing on stdout and a plain-text error on stderr +naming that entry. No scenario checks the +listing's order: scenario 23's two runtimes share one install time, and the +registry is read in filesystem order, which can match key order by chance. +The order is pinned by unit tests: +`runtimes_list_json_has_exactly_the_documented_fields_newest_install_first` +plants five runtimes with one install time in reverse key order beside older +and newer ones, and `registry_sort_is_newest_install_first_then_by_key` tests +the sort on its own. + +Which runtimes the listing marks `active` and `rollback_target`, and when it +reports `active_problem`, are in +[json-output.md](json-output.md#which-runtimes-are-marked): the listing asks +the resolver `rocm examine`, `rocm runtimes activate`/`rollback`, +`rocm runtimes uninstall` and `rocm update` use, and the surfaces that still +resolve differently are listed there. +`@id:runtime-lifecycle-list-default-id-in-other-letter-case` +(`runtime-lifecycle-27`) sets the first runtime's id in upper case through +`rocm config set-default-runtime` and checks that the JSON marks that runtime +active with `active_problem` `null`, and that the text marks it `*` with no +`active_status:` line. +`@id:runtime-lifecycle-list-active-key-in-other-letter-case` +(`runtime-lifecycle-28`) edits the config's active runtime key to the first +runtime's key in upper case while the default id still names the second, and +checks that both forms mark the first runtime active with no problem and that +`rocm examine --json` reports the first runtime's folder as +`active_runtime_root`. +`@id:runtime-lifecycle-list-dead-active-key-falls-back-to-default` +(`runtime-lifecycle-29`) edits the key to one no runtime has while the default +id still names the second runtime, and checks that both forms mark the second +runtime active, that the JSON reports `missing_manifest_for_key` and the text +the matching `active_status:` line, and that `rocm examine --json` reports the +second runtime's folder. +The `listing_state_*` unit tests in +[runtimes_cmd.rs](../apps/rocm/src/runtimes_cmd.rs) list a state matrix in +memory, one test per row (no config; an active key exact, in other case, or naming nothing with +the default id resolving one, none or several; no active key with the default +id matching none, one, one in other case, two, or two differing in case; keys +differing only in case; a previous key exact, in other case, given as a runtime +id, with whitespace, naming nothing, ambiguous, or naming one of two case twins +exactly; and a dead key falling back to the rollback target). Each checks the +JSON flags, `active_problem`, the text markers and the `active_status:` line, +and that `active` and `rollback_target` are what `current_runtime_manifest` and +`select_runtime_manifest` resolve. Keys differing only in case are built in +memory, so these tests hold on a case-insensitive filesystem too. +`runtimes_list_json_writes_a_non_utf8_path_lossily` (Unix only) lists a data +folder whose path is not valid UTF-8, and +`runtime_toolchain_text_and_json_words_agree_for_every_variant` holds the +`toolchain` words in the text and the JSON together: + +```bash +cargo xtask e2e -- -n "runtime-lifecycle-(07|23|24|25|26|27|28|29|30|31) " +cargo test -p rocm --bin rocm runtimes_list +cargo test -p rocm --bin rocm listing_state +cargo test -p rocm --bin rocm runtime_toolchain_text_and_json_words_agree_for_every_variant +cargo test -p rocm --bin rocm registry_sort_is_newest_install_first_then_by_key +``` + +Each runtime's row is followed by its `install_root:` and a `provenance:` line +naming the `channel`, the package `source` (the wheel index, or the downloaded +archive for a tarball install), the `source_layout_generation` and the exact +`device_target` its manifest recorded, for example: + +```text + provenance: channel=nightly source= source_layout_generation=next-v1 device_target=gfx1201 +``` + +The values are the ones the manifest recorded: the package index, else the +artifact URL (never the marker an adopted runtime keeps there); the layout +field, else the wheel composition's; the recorded target, else the +`device-` extra in the package specs. For the layout and target this +chooses the same recorded field `rocm update` uses, through the same accessors; +`rocm update` then uses the value as recorded, while the listing trims it and +prints a blank one as `unrecorded`. Each step moves on only when a field is +absent, so a blank field is not skipped in favour of the next one, and a fact +recorded nowhere also prints `unrecorded`. Nothing is guessed from other +facts: a family-wide or tarball install shows `device_target=unrecorded` rather +than a target guessed from its family, and a runtime registered with +`rocm runtimes adopt` shows `source=unrecorded`. The chains, their precedence, +the blank and trimming rules are pinned by +`runtime_list_reads_each_provenance_fact_from_where_it_was_recorded`; blank or +padded values are not what `rocm install sdk` writes, but can come from a +hand-edited manifest or one brought in with `rocm runtimes import`. An adopted +runtime's channel is the one adoption recorded: the channel `--channel` or +`--runtime-id` named, `release` when neither names one. All three paths +(neither flag, `--channel nightly`, and `--runtime-id therock-nightly:` +with no `--channel`) are pinned by +`adopted_runtime_lists_its_channel_and_its_source_as_unrecorded` in +[runtime_adopt.rs](../apps/rocm/src/runtime_adopt.rs). The marker adoption +writes in place of an artifact URL is persisted manifest format. Code in the +crate refers to it only through its constant, so two checks pin the literal +value: `adoption_marker_keeps_its_persisted_value` in +[runtime_registry.rs](../apps/rocm/src/runtime_registry.rs), and +`runtime-lifecycle-22` (below), which plants it in an adopted runtime's +manifest. +`rocm examine` prints the same four facts for the active runtime as +`active_runtime_channel`, `active_runtime_source`, +`active_runtime_source_layout_generation` and `active_runtime_device_target`, +from the same fields. The unit tests +`runtime_list_names_where_each_runtime_came_from` and +`examine_names_the_active_runtime_provenance_the_listing_shows` in +[runtimes_cmd.rs](../apps/rocm/src/runtimes_cmd.rs) pin the values, the +latter for an active nightly runtime and an active adopted one; +`@id:runtime-lifecycle-list-names-runtime-provenance` (`runtime-lifecycle-22`) +checks the listing end to end with a planted nightly runtime, an older one +that recorded neither its layout nor an exact target, and an adopted one whose +manifest holds the adoption marker as earlier releases wrote it, which must +list `source=unrecorded`; and +`@id:examine-reports-where-the-active-runtime-came-from` (`examine-54`) checks +the `examine` lines. Both plant read-only manifests and run on the mock lane: + +```bash +RUSTC_WRAPPER= cargo xtask e2e -- -n 'runtime-lifecycle-22 |examine-54 ' +``` + +`rocm runtimes list --json` gives each runtime the same four facts as a +`provenance` object under the same names, with `null` where the text prints +`unrecorded` ([json-output.md](json-output.md)). Both +forms render from one listing, and the JSON takes its names and order from +`RuntimeProvenance::recorded_fields`, the list `RuntimeProvenance::fields` +shows. `runtimes_list_json_names_recorded_provenance_and_null_for_unrecorded` +in [runtimes_cmd.rs](../apps/rocm/src/runtimes_cmd.rs) pins the whole object +for a nightly, an older, an adopted, a tarball, a padded and an all-blank +runtime; `runtimes_list_text_and_json_name_the_same_provenance` checks over the +same runtimes that the text line and the JSON object name the same facts with +the same values; and +`runtimes_list_json_has_exactly_the_documented_fields_newest_install_first` +pins the object's key set. +`@id:runtime-lifecycle-list-json-names-runtime-provenance` +(`runtime-lifecycle-31`) plants the runtimes of `runtime-lifecycle-22`, reads +each one's provenance from the JSON, and compares it with the text listing of +the same state. It sets `ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK`, as the +examine scenarios that plant a runtime do: with wheel runtimes registered, the +startup update check looks for a Python and the network and can print a +progress line on stdout ahead of the document. It runs on the mock lane: + +```bash +cargo xtask e2e -- -n 'runtime-lifecycle-31 ' +cargo test -p rocm --bin rocm runtimes_list ``` Activate one validated runtime: @@ -1844,6 +2119,17 @@ and `config set-engine --runtime-id` store a selector without resolving it. For `runtimes activate`, `@id:runtime-lifecycle-case-twin-selector-refused` also checks that nothing became active and that naming one key exactly then works. +Runtimes installed before and after the ROCm 10 layout stay usable side by +side: `@id:runtime-lifecycle-side-by-side-across-packaging-change` lists and +activates a ROCm 7 runtime, a ROCm 10 release runtime and a ROCm 10 nightly +runtime without rewriting their install records, +`@id:runtime-lifecycle-side-by-side-uninstall-keeps-the-others` uninstalls one +and checks the others keep their folders and records, and +`@id:runtime-lifecycle-side-by-side-shared-name-refused` checks that the runtime +id the two release runtimes share is refused with both keys named. +Uninstalling a runtime that is the rollback target clears it as the rollback +target in both the config and the active-runtime marker. + Developer-only previous-runtime regression check: ```bash @@ -1904,6 +2190,7 @@ Focused unit coverage: ```bash cargo test -p rocm --bin rocm runtime_activation +cargo test -p rocm --bin rocm runtimes_list cargo test -p rocm --bin rocm runtime_import cargo test -p rocm --bin rocm runtime_adopt cargo test -p rocm --bin rocm runtime_install_writes @@ -1922,7 +2209,6 @@ cargo test -p rocm-core model_recipe cargo test -p rocm-core staleness_warning cargo test -p rocm-core load_model_recipe_index cargo test -p rocm --bin rocm update_report_policy -cargo test -p rocm-engine-lemonade stdio_protocol_routes_all_methods_without_side_effects cargo test -p rocm-engine-vllm stdio_protocol_routes_all_methods_without_side_effects cargo test -p rocm-engine-vllm cargo test -p rocm-engine-aim @@ -1931,6 +2217,43 @@ cargo test -p rocmd event_collector cargo test -p rocmd event_dispatcher ``` +## vLLM Tensorizer Check + +vLLM's wheel pins a tensorizer release that cannot be imported under Python 3.14, +and `rocm engines install vllm` checks that it imports and upgrades it when it +does not (see [vllm.md](vllm.md)). The `vllm-tensorizer-NN` E2E scenarios +([vllm_tensorizer.feature](../tests/e2e-cucumber/features/vllm_tensorizer.feature); +gated only on `@requires-os:linux`, with no GPU, network or nightly tag, so the +Linux e2e jobs run them) run that command against a planted managed runtime +whose Python cannot import the pinned tensorizer. They check the upgrade +warning, `dependency_check: expected_divergence`, the note's tensorizer release +and the release the runtime ends up holding in the same scenario; that a second +install neither upgrades again nor reinstalls vLLM; that an upgrade which does +not help warns with the `KeyError` shown when this run did not install vLLM and +refuses, saying the install ran, when it did (`--reinstall`); that a coloured traceback and a +`PYTHONPATH` only the server's launch environment provides are judged correctly; +that a missing library, or a non-library exception raised below tensorizer, is +left alone; that an upgrade which cannot be fetched warns when this run did not +install vLLM but fails when it did (a first install or `--reinstall`; a repair takes +the same path); and +that repairing a genuine violation beside the upgraded tensorizer clears it and +upgrades tensorizer again. Unit tests in [install.rs](../engines/vllm/src/install.rs) and +[repair.rs](../engines/vllm/src/repair.rs) cover the classification (with Unix and Windows paths), the probe +command, the probe and upgrade timeouts and the repair decision +(`cargo test -p rocm-engine-vllm -- install::tests repair::tests`, which also +runs the other tests of those two modules; a `tensorizer` name filter would miss +the probe-command, timeout and hung-child tests). The shared rule that decides what +counts as vLLM's own unmet tensorizer pin is tested in `rocm-core` +(`cargo test -p rocm-core tensorizer_override`), and its use in the dependency +check report in +[runtime_install_finalize.rs](../apps/rocm/src/runtime_install_finalize.rs) +(`cargo test -p rocm --bin rocm runtime_install_finalize`). +To run just these locally: `cargo xtask e2e -- -n 'vllm-tensorizer'`. The `-n` form +selects by scenario name and bypasses the harness's own gating, so it needs a Linux +host for these scenarios. +The scenarios are Linux only; the real Python 3.14 import is only exercised by +hand ([manual-testing.md](manual-testing.md#6c-the-tensorizer-upgrade-on-python-314)). + ## Lemonade Installs On Disk `features/lemonade_installs.feature` covers what a user is told about the @@ -1945,6 +2268,31 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the disk that nothing was downloaded. That variable also waives serve's no-GPU refusal, as serve-18's failure hook does. +- `lemonade-installs-07` (Linux) reaches the refusal of a host whose glibc + or OpenSSL falls short of what the pinned `lemond` needs. It reports an + Ubuntu 20.04-like host through `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and + `ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` (same `rocm/e2e-test-hooks` build): + glibc 2.31, and no `libssl.so.3` in the loader cache because only OpenSSL 1.1 + is installed. It checks that the message names what each library needs and + what the host has, and that nothing was downloaded. The probe itself, + including a planted `ldconfig -p` that lists `libc.so.6` but no + `libssl.so.3`, and refusing to start `lemond` on such a host are unit-tested + in [host_libraries.rs](../engines/lemonade/src/host_libraries.rs) and + [process.rs](../engines/lemonade/src/process.rs). No scenario reaches the + start-time refusal: `rocm serve` runs the install check first, a Linux serve + with a packaged `llama-server` never starts `lemond`, and the mock lane has + no GPU for serve to resolve. +- `lemonade-installs-08` (Linux) sets the same old-host hooks plus + `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`, and reports the drive + nearly full. It checks that the shortfall is printed as a warning naming the + variable, and that the install went on past the check to the space refusal + without downloading anything. The opt-out on the `lemond` start is + unit-tested in [process.rs](../engines/lemonade/src/process.rs). +- `lemonade-installs-09` (Linux) plants a tree of the pinned Lemonade in the + data folder, reads the pinned version from a first refused run, and re-runs + the install on the old-host hooks. It checks that the refusal says the + installed copy stays as it is rather than "Nothing was downloaded.", and that + every file of the planted tree is unchanged. - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record @@ -2250,15 +2598,37 @@ on a non-default `--host` is still required: automatic selection only probes A loopback bind (`rocm serve `, default `127.0.0.1`) stays credential-free. A public bind (`--host 0.0.0.0 --port --allow-public-bind`) requires an API key: pass one with `--api-key` / the `ROCM_SERVE_API_KEY` environment -variable, or let one be generated. The key is handed to the engine via a 0600 -key file (never argv), persisted as a 0600 per-service file (not the OS keychain, -which is unavailable on headless serving hosts), and printed once as client +variable, or let one be generated. The key is handed to the engine via a key +file (never argv), persisted as a per-service file (not the OS keychain, which +is unavailable on headless serving hosts), and printed once as client configuration — it must never appear in `rocm services`, `rocm logs`, or the -audit log. Verifying that the running server actually *rejects* an -unauthenticated request requires a live engine; that end-to-end assertion is a -deferred follow-up and is **not yet** exercised by the GPU acceptance scripts -(`scripts/vllm_therock_gpu_test.py`). The unit tests cover key policy, -generation, the 0600 key file, and key-file resolution. +audit log. Only the serving user can read the key file: it is mode 0600 on Unix +and has a protected DACL granting only that user on Windows, and every store +replaces it with a freshly created owner-only file, so permissions loosened in +place do not survive a re-store. Verifying that the running server actually +*rejects* an unauthenticated request requires a live engine. For vLLM that is +the `@nightly` GPU scenario `serve-vllm-key-guards-every-route` (serve-61), +which runs in the nightly runs on Instinct GPUs (see "vLLM endpoint-key +coverage" below). For the other engines, AIM included, no test yet asserts that +the running server refuses a keyless request. The unit tests cover +key policy, generation, the owner-only key file (including one whose +permissions were loosened before a re-store, and on the Windows build job the +DACL read back from disk), and key-file resolution. The tests that fail +without the repair of a loosened key file are `restoring_a_key_over_a_loosened_file_makes_it_0600_again` +in [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), `write_owner_only_file_atomically_tightens_a_loosened_file` +in [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), and on Windows +`write_owner_only_file_atomically_replaces_a_loose_dacl_with_an_owner_only_one` +in [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), which starts from a destination whose DACL grants Everyone. `serve-59` (`@id:serve-endpoint-key-readable-by-owner-only` +in [model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature)) +guards fresh-create behaviour only: it checks that a real serve's newly +written key file is mode 0600 on the Linux GPU lanes, which held before this +repair too, so it is a guard against a regression in that path, not evidence +for the repair. + +```bash +cargo test -p rocm-core --lib owner_only +cargo test -p rocm --bin rocm endpoint_keys +``` The same requirement holds across respawns, because the invariant belongs to the recorded *host*, not to the key file: a service recorded on a public host refuses @@ -2284,15 +2654,29 @@ service id. `rocm services remove` / `rocm services prune` are the one place a key file is dropped for a service that was never stopped: the record itself is being -deleted, so keeping its key would strand a 0600 secret belonging to a service -that can no longer be restarted. Neither will touch a record the liveness +deleted, so keeping its key would strand an owner-only secret belonging to a service +that can no longer be restarted. They also delete any copy of the key a save +killed part-way left beside it (`.endpoint-key.tmp---`): +`remove` with its record, and `prune` with its record or, when the record is +already gone, as a leftover under the same age rule. That copy is recognised +only by the exact name the shared atomic writer gives it, so another service's +key whose id happens to contain `.tmp-` is never taken for one. Neither will touch a record the liveness refresh still reads as running, and `prune` re-checks liveness immediately before each delete — but the check and the delete are not atomic, so that narrows the race against a concurrent `restart` rather than eliminating it. -`features/service_record_cleanup.feature` covers both commands on the mock lane, -asserting on the files left on disk rather than on the summary line the command -prints — the defect they exist to fix is a file being left behind, which a -summary claiming success cannot reveal. +`features/service_record_cleanup.feature` covers both commands on the mock lane. +Its scenarios assert mainly on the files left on disk, because the defect they +exist to fix is a file being left behind, which a summary claiming success +cannot reveal; where they read the output it is for what the user must be +told, such as the log path before a removal. `service-cleanup-01` plants an +interrupted-save key copy beside a record and checks `remove` deletes it and +counts it in `files removed:`; `service-cleanup-03` plants one for the record +`prune` removes and checks it goes with that record, and one for a record +already deleted by hand and checks `prune` previews and sweeps it. The unit +test `services_prune_takes_a_pruned_records_staged_endpoint_key` in +[service_records.rs](../apps/rocm/src/service_records.rs) covers the first of +those too, and its neighbours there cover the age rule and what the sweep +leaves alone. Every command that takes a service id (`services stop`/`restart`/`logs`/`remove` and `rocm logs --service`) reads the record through one loader, @@ -2308,7 +2692,7 @@ error names to confirm it works and does not list the id. `prune`'s leftover sweep carries a second, unrelated race: a file with no record beside it is also what a *launch in progress* looks like, because `rocm serve` -writes the 0600 endpoint key before it writes the record, and `--any-age` leaves +writes the owner-only endpoint key before it writes the record, and `--any-age` leaves no age rule to hide that window behind. `serve` already holds the shared managed-launch lock across both writes, so `prune` acquires the same lock before it reads the directory. @@ -2364,8 +2748,9 @@ via `spawn_hidden_console_with_log`, whose env-override API is path-valued only, so it cannot receive the value-typed `LEMONADE_API_KEY` that Lemonade's server reads. Rather than launch an unauthenticated public server, `rocm serve` **fails closed** on that exact combination (public bind + `--engine lemonade` + Windows) -and directs the user to `--engine vllm` or a loopback host. vLLM (all platforms, -via `VLLM_API_KEY`), direct llama-server serving (via `--api-key-file`), +and directs the user to `--engine vllm` or a loopback host. vLLM (Linux and WSL, +via `VLLM_API_KEY` plus the `rocm_cli_vllm_auth` middleware described below), +direct llama-server serving (via `--api-key-file`), and Lemonade on non-Windows are all fully supported. Teaching the Windows detached-spawn primitive to carry a value-typed override (so Windows Lemonade can serve public with auth) is tracked as a separate follow-up. @@ -2390,6 +2775,84 @@ lemonade` against the shared tree) reaches the restore only on a runner whose shared tree still holds a stale pin; on a healthy runner the install succeeds and the restore never runs, so a green lane does not show that it works. +vLLM endpoint-key coverage: `VLLM_API_KEY` on its own makes vLLM check the key +only under a fixed set of path prefixes (`/v1` and a few others), while the +server also answers routes such as `/invocations`, `/tokenize` and `/metrics`. +So a keyed vLLM launch also passes `--middleware +rocm_cli_vllm_auth.RequireEndpointApiKey`, an ASGI middleware compiled into the +engine adapter +([rocm_cli_vllm_auth.py](../engines/vllm/src/rocm_cli_vllm_auth.py)), written +under the engine's data directory at launch and put on the server's +`PYTHONPATH`. It +requires the key on every route except `GET`/`HEAD /health` and CORS +preflights, and raises instead of constructing when the key is empty, so a +server that loaded it never answers a request unchecked. + +```bash +cargo test -p rocm-engine-vllm endpoint_auth +cargo test -p rocm-engine-vllm vllm_serve_args +``` + +The `endpoint_auth` tests run the shipped middleware under `python3` with a +standard-library ASGI harness (they fail, rather than skip, when `python3` is +missing on Linux), so the per-route behavior is covered on every PR without a +GPU. The `vllm_server_command` tests assert, on every PR, that a keyed launch +gets the `--middleware` argument, `VLLM_API_KEY`, and the middleware directory +at the front of `PYTHONPATH` ahead of the runtime's own entry, together, and +that an unkeyed launch gets none of them: + +```bash +cargo test -p rocm-engine-vllm vllm_server_command +``` + +What no unit test covers is the call in `spawn_vllm_server` that hands the +resolved endpoint key to `vllm_server_command`: that spawn path probes the host +(OpenMPI, libatomic, libnuma) before it builds the command, so only a real +launch exercises it. + +That a real vLLM server loads the middleware and refuses keyless calls is +covered by the `@nightly` GPU scenario `serve-vllm-key-guards-every-route` +([model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature)), +which runs in the scheduled nightly runs on Instinct GPUs, where vLLM is the +serving engine: the MI300X job in [nightly.yml](../.github/workflows/nightly.yml) +and the shared GPU queue's Instinct lanes, MI350P among them +([nightly-on-the-queue.md](nightly-on-the-queue.md)). Those run after merge, not +per PR. It is not tagged `@merge-queue`: +the merge queue runs no lane where vLLM is the engine (the MI300X lane does not +run on `merge_group`), so under that tag it would run nowhere. To run it on a +branch before merge, dispatch `e2e-selfhosted.yml` on the MI300X lane with a +`name_filter`, which bypasses the harness's tag gates (`@nightly` included), so +the scenario runs without `include_nightly`: + +```bash +gh workflow run e2e-selfhosted.yml --ref \ + -f platform=app-dev-gpu -f name_filter='^serve-61 ' +``` + +Lemonade backend selection and alignment focused tests (reading the +`llamacpp` rows out of `lemonade backends` when other recipes have indented +rows of their own, and the second alignment attempt choosing the newest +`lemonade-sdk/llama.cpp` release that ships the archive for the active ROCm +version rather than the newest release, and reading Lemonade's +`update_required`/`update_available` backend states, and skipping a warning +line inside the table, even one with a state word in it): + +```bash +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::parse +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::another_recipes_backend +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::update_states +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::a_warning +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::a_field_wider +cargo test -p rocm-engine-lemonade backend_alignment::tests::tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::align_falls_through_to_tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::llamacpp_asset_name +``` + +They run on fixtures: the `lemonade backends` table Lemonade 2026.44.1 prints, +and a release listing shaped like the real one. The live paths (a real +`lemond`, the GitHub listing, a real backend install) run only on the gated +Lemonade GPU lanes. + Engine recipe adapter contract focused tests: ```bash @@ -2846,23 +3309,50 @@ a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd suite's own host filter, so it would also run the Windows-only `service-stop-07` on a Linux host, where it fails for want of PowerShell. -`features/engine_ipc.feature` (`engine-ipc-01` and `-02`, Linux, mock lane) +`features/engine_ipc.feature` (`engine-ipc-01` to `-12`, Linux, mock lane) covers how the daemon reads an engine child's answer, through the same -`stop_server` tool. Each scenario copies the real `rocmd` into a scratch -directory and puts a stand-in `rocm` shell script beside it, which is the +`stop_server` tool (`-01` to `-07`), and how an engine reads a request +(`-08` to `-12`). Each of `-01` to `-07` copies the real `rocmd` into a +scratch directory and puts a stand-in `rocm` shell script beside it, which is the executable the standalone daemon runs for an engine request; the stand-in records its argv and the request it was sent, and prints what the scenario -says. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the +says. Each of `-08` to `-12` runs the real `rocm __engine-stdio vllm`, writes +the scenario's request to its stdin, and reads what it prints and its exit +status. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the report carries the engine's answer (not an error built from the exit status). `engine-ipc-02` has it print something that is not an answer, write to stderr and exit non-zero: the report's engine error is the parse failure, naming the -exit status the engine ended with, followed by its stderr. Both check the report together with the server's process, record and -endpoint key, and that the stand-in was run once as `__engine-stdio vllm` with -one `stop` request line. The server is a `vllm` one, for which the engine's -`Stop` is advisory, so in `-02` the stop still completes; an engine whose stop +exit status the engine ended with and what was wrong with its stdout, followed +by its stderr. `engine-ipc-03` has the engine answer that it stopped, in an engine +protocol this release cannot use (`9.0.0`): the report's engine result is the +refusal naming both versions, not the answer. `engine-ipc-04` has it write a log +line on stdout before a valid answer: the report's engine error says it wrote +other output before its answer, quoting the line. `engine-ipc-05`, `-06` +and `-07` have it answer in this release's protocol, in a newer one that only +adds to it (the next patch, computed from `ENGINE_PROTOCOL_VERSION`, with a +field this release does not know in the envelope and in the stop's data), and with +no version at all, as an engine from before the +field does: each is read as answered. (They are three scenarios, not one +outline, because the suite keys each scenario's result by its `@id`.) Every one checks the report together with the server's +process, record and endpoint key, and that the stand-in was run once as +`__engine-stdio vllm` with one `stop` request line, written in this release's +protocol. The server is a `vllm` one, for which the engine's +`Stop` is advisory, so in `-02` to `-04` the stop still completes; an engine whose stop decides the confirmation (`aim`) is not staged here, because teardown would -then run the real engine's stop. Run them with a hooks build and `-n -"engine-ipc"`. +then run the real engine's stop. `engine-ipc-08` runs the real `rocm +__engine-stdio vllm` and sends it a `stop` request made for protocol `9.0.0` for a +server the engine's own state file names (a live `sleep`, with its `/proc` start +time): the answer is the `incompatible_protocol` refusal naming both versions, +the process is still running and the engine's record still names it, where a +`stop` the engine served would have ended it. `-09` sends a method no engine +has (`frobnicate`) at `9.0.0`, and `-10` a stop whose `protocol_version` is the +number `9`: each is answered `incompatible_protocol`, which only reading the +version before the method or the rest of the envelope can produce, and `-10` +also shows the server still running. `-11` sends `frobnicate` in this release's +protocol (`unknown_method`) and `-12` text that is not JSON (`invalid_request`). +Each of `-08` to `-12` pins the whole answer and an exit status of 0. Run them +with a hooks build and `-n "engine-ipc"`; the steps take this release's version from +`rocm_engine_protocol::ENGINE_PROTOCOL_VERSION`, a dev-dependency of the suite. Manual restricted-tool smoke: @@ -2891,6 +3381,38 @@ approval. User-facing TUI/chat flows should still route mutating tool calls through their normal approval cards rather than relying on this hidden helper flag. +Assistant tool-call arguments: + +The approval card for a mutating assistant tool call and the command its +approval runs are built from the same argument object. No tool takes a +parameter named `arguments`, so a call whose arguments carry a nested +`arguments` field is refused by `validate_chat_tool_call` in +[tool_policy.rs](../apps/rocm/src/tool_policy.rs) before any card is shown, on +the dash, chat and `rocm mcp-call` paths alike. `chat-17` +(`chat-ambiguous-tool-call-refused-before-approval` in +[chat.feature](../tests/e2e-cucumber/features/chat.feature), Linux mock lane, +no GPU) drives it through the interactive chat: a mock model answers with a +two-payload `launch_server` call, the model is told the call was refused for +its nested field, and no approval prompt appears. + +The hidden `rocm mcp-call` command still accepts a pasted MCP `tools/call` +params envelope (`name`, `arguments` and an optional `_meta`) and unwraps it at +its own boundary, but only a pure envelope, and it refuses one that names a +different tool. `chat-18` runs a read-only tool from such an envelope and +checks the tool's own result; `chat-19` passes tool fields mixed with a nested +`arguments` object and checks the call is refused and prints no tool result. +Arguments that are not a JSON object are refused rather than run as `{}`. + +```bash +cargo xtask e2e -- -n "chat-1[789]" +cargo test -p rocm --bin rocm nested_arguments +cargo test -p rocm --bin rocm seam_approval_card_matches +cargo test -p rocm --bin rocm mcp_call_ +cargo test -p rocm --bin rocm no_rocm_tool_declares_an_arguments_parameter +cargo test -p rocm-dash-tui no_rocm_tool_declares_an_arguments_parameter +cargo test -p e2e-cucumber --lib a_scripted_tool_call_answers_only_the_triggering_user_turn +``` + Artifact cache and prefetch policy tests: ```bash @@ -3132,6 +3654,64 @@ is registered for Linux, Windows and WSL and only adds to that score. [host.rs](../crates/rocm-core/src/diagnose/host.rs) holds that for all three families. +When a fix's own command does not succeed, `rocm fix` exits 4 and prints one +line on stderr, built from how the command ended +(`Step` in [fix/consent.rs](../crates/rocm-core/src/fix/consent.rs)): + +| How the command ended | What the line says | +|---|---| +| exited non-zero, or could not be started | it exited `N` (or could not be started), and the change was NOT made | +| timed out (60s for a change), or was stopped by a signal or a failed wait | it was stopped, the change may or may not have been made, how to check it (`Check with` a shell command on Linux, `Check in PowerShell with` an expression on Windows), and the `rocm fix` command to re-run | +| a Windows read of the current value did not complete (8s) | it could not read the current value, and nothing was changed; the runner stops before its preview of the change or any prompt, under `--dry-run` too; the catalog entry printed before it still appears | + +Each row for each runner is unit-tested with an injected command runner in +[fix/runners.rs](../crates/rocm-core/src/fix/runners.rs) and +[fix/consent.rs](../crates/rocm-core/src/fix/consent.rs), on every host. The +scenarios in [diagnose.feature](../tests/e2e-cucumber/features/diagnose.feature) +run the real binary: `diagnose-15` (fix-4's helper exits 1) and `diagnose-68` +(it never finishes) stand a fake `sudo` on a restricted `PATH` that records the +arguments it was given, and assert the line together with the exit code, that +the helper was asked to add the user to the groups, and that the success line is +absent. Both are `@requires-os:linux @requires-bare-metal`. `diagnose-68` +waits out the real one-minute bound; there is no switch to shorten it. The step +waits for the CLI on a blocking thread (`run_rocm_with_scenario_env_off_executor` +in [e2e.rs](../tests/e2e-cucumber/tests/e2e.rs)): the whole suite is polled on +one thread, so a synchronous wait would stall every concurrent scenario for that +minute and push the TUI scenarios past their wall-clock caps. + +On Windows, `diagnose-39` (fix-6), `diagnose-69` (fix-2) and `diagnose-70` +(fix-9 with `--device-index 1`, run as a `--dry-run`) give the CLI an empty +`PATH`, so its PowerShell read cannot start. They assert exit 4 and the +read-failure line, and that the runner went no further: no `Plan: ...` preview +of the change for fix-6 and fix-9, and no report that fix-2's variable is +absent. The catalog entry `rocm fix` prints before the runner still appears. The +empty `PATH` denies PowerShell only: Rust's `Command` looks for a bare program +name in the CLI's own directory, System32 and the Windows directory before +`PATH`, so `setx.exe` (in System32) is still found, while `powershell.exe` (in a +System32 subdirectory) is not. None of the three can write the runner's real +User environment even if the read guard regressed: fix-6 writes through +PowerShell, which cannot start; fix-2 would read both scopes as unset and find +nothing to clear, so it never reaches its `setx`; and `diagnose-70` is a dry +run, which never runs `setx`. The fix-9 path that does write is held by the unit +tests. These three run only on the self-hosted Strix Halo Windows lanes, neither +of which is a required check: `E2E tests (Strix Halo, Windows, )` +(`e2e-gpu-strix-windows` in +[e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), on a pull request +that touches its `serve` paths, on pushes and in the merge queue) and `E2E tests +(Strix Halo, Windows, incl. nightly-only)` (`e2e-gpu-nightly-strix-windows` in +[nightly.yml](../.github/workflows/nightly.yml)). The required hosted Windows +lane (`windows-release` in [ci.yml](../.github/workflows/ci.yml), behind +`windows-build-and-test`) runs only `@lifecycle` scenarios and the ids in its +`E2E_ALSO_RUN_IDS`, which do not include them. The Linux lanes compile them. No +scenario times out a Windows `setx` or PowerShell write: those rows are covered +by the unit tests. To run the two Linux scenarios on a bare-metal Linux host +(`-n` skips the tag gates, so on WSL2 they fail at the platform refusal +instead): + +```bash +cargo xtask e2e -- -n '(diagnose-15|diagnose-68) ' +``` + Provider audit focused test: ```bash @@ -3283,11 +3863,13 @@ The user-visible side is `serve-55` (`@id:serve-small-assistant-short-name`): It carries `@requires-gpu` and deliberately no `@requires-engine`, because the engine is the CLI's to pick here — so the lane that exercises the Lemonade half is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on -the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, -MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests -([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` -path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, -so the two unit tests above are the blocking per-PR coverage. The scenario +the queue nightly only) and the vLLM half falls to the Instinct lane. Of all +these lanes only MI300X (two checks, `e2e-gpu-queue` on the shared GPU queue +and `e2e-gpu` on a self-hosted runner, the same silicon) and the Strix Halo +Windows and WSL2 lanes run on pull requests ([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when +the `serve` path filter matches; W7900 has no per-PR lane any more and runs only +on the queue nightly), and none of their check names is a required status +check, so the two unit tests above are the blocking per-PR coverage. The scenario carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares their known serve failure there has not been observed, and a row claiming a @@ -3485,6 +4067,15 @@ python scripts/nightly_canary.py --self-test What the scheduled canary checks, who is told and how to rerun it are in [ci-hardware-testing.md](ci-hardware-testing.md#nightly-stream-canary). +The vLLM route check (advisory) has a self-test of the same kind: + +```bash +python scripts/vllm_route_check.py --self-test +``` + +What it checks and why it is not a required check are in +[ci-hardware-testing.md](ci-hardware-testing.md#vllm-route-check-advisory). + The ROCDXG digest check pins the shell fragment the WSL driver plan generates as a whole string — so a change to its quoting, spacing or field order fails the test rather than slipping past a substring assertion — and then runs that @@ -3560,8 +4151,10 @@ inputs come from the environment for parity with those scripts: ### Install lifecycle (opt-in `@lifecycle` E2E) The full install lifecycle — packaging, signature-verified install, tamper -rejection, reinstall, PATH handling, and uninstall — is expressed as -`@lifecycle` scenarios in the [E2E cucumber suite](../tests/e2e-cucumber/README.md) +rejection, reinstall, PATH handling, uninstall, and how download progress is +drawn (one bar per download that stays inside the terminal, a terminal too +narrow for a bar saying so rather than going silent, and the wget fallback) — +is expressed as `@lifecycle` scenarios in the [E2E cucumber suite](../tests/e2e-cucumber/README.md) (`features/install_lifecycle.feature`). They drive `cargo xtask package`, the real root installer (`install.sh` / `install.ps1`), and the installed binaries end to end, replacing the former @@ -3638,30 +4231,41 @@ needs a non-root runner. `@id:lifecycle-linux-uninstall-keep-binaries-keeps-path The `windows-release` job of the required `windows-build-and-test` lane additionally folds in the hermetic `comfyui-01`/`comfyui-02`/`comfyui-03` -scenarios and the native-Windows vLLM refusal scenarios -`serve-33`/`serve-34`/`serve-35` (none of them `@lifecycle`) via `E2E_ALSO_RUN_IDS`, a -comma-separated list of `@id`s: +scenarios, the native-Windows vLLM refusal scenarios +`serve-33`/`serve-34`/`serve-35`, and the side-by-side runtime scenarios +`runtime-lifecycle-19`/`runtime-lifecycle-20`/`runtime-lifecycle-21` (none of +them `@lifecycle`) via `E2E_ALSO_RUN_IDS`, a comma-separated list of `@id`s: ```powershell $env:E2E_INCLUDE_LIFECYCLE = "1" $env:E2E_ONLY_LIFECYCLE = "1" -$env:E2E_ALSO_RUN_IDS = "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows" +$env:E2E_ALSO_RUN_IDS = "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows,runtime-lifecycle-side-by-side-across-packaging-change,runtime-lifecycle-side-by-side-uninstall-keeps-the-others,runtime-lifecycle-side-by-side-shared-name-refused" cargo xtask e2e ``` -These three were made hermetic and Windows-portable without being tagged -`@lifecycle` — they're ComfyUI install scenarios, not release packaging/ -install/uninstall ones, and tagging them `@lifecycle` would drop them out of -the full-suite runs in `nightly.yml` and `e2e-selfhosted.yml`, neither of -which sets `E2E_INCLUDE_LIFECYCLE`. `E2E_ALSO_RUN_IDS` selects them into this -lane instead, so this is the one required lane that guarantees Windows -coverage for them; the self-hosted Windows lane that otherwise runs them is -optional. +All nine are hermetic (planted fixture binaries or runtimes, no GPU, no +network) and Windows-portable without being tagged `@lifecycle` — they are +ComfyUI install, vLLM refusal and runtime-selection scenarios, not release +packaging/install/uninstall ones, and tagging them `@lifecycle` would drop +them out of the full-suite runs in `nightly.yml` and `e2e-selfhosted.yml`, +neither of which sets `E2E_INCLUDE_LIFECYCLE`. `E2E_ALSO_RUN_IDS` selects them +into this lane instead, so this is the one required lane that guarantees +Windows coverage for them; the self-hosted Windows lane that otherwise runs +them is optional. Heavy CI runs the matching set per platform. The scenarios reject untrusted signers, bad checksums, bad detached signatures, and missing required signature -sidecars before activation; assert the installer's get-started block (banner art -only on a UTF-8 terminal), `!!` failure marker, and absence of ANSI escapes in +sidecars before activation. On Linux, with no public key supplied and no +signature asked for, an install of a specific version with +`ROCM_CLI_REQUIRE_SIGNATURE=0` still rejects an archive signed with a local key, +which the pinned release key does not accept (lifecycle-36), and a release +install with the variable unset still rejects an archive with no `.sig` +(lifecycle-37). The missing-`.sig` case with nothing asked for is covered on +the `release` channel only. Both scenarios are Linux-only: on Windows, the +same default path (no key supplied, no signature asked for) has no scenario, +and lifecycle-17 covers a no-key install only with the requirement set. The +scenarios also assert the installer's get-started block (banner art only on a +UTF-8 terminal), `!!` failure marker, and absence of ANSI escapes in piped or `NO_COLOR` output (the colored TTY branch is not driven by any scenario); cover generated private-key PEM packaging and generated public-key PEM installer verification; verify first-install PATH setup @@ -3678,9 +4282,11 @@ the installer's record, and the running Windows `rocm.exe` removed. The two Windows scenarios that change the user `PATH` are `@serial`, since it is one registry value for the whole machine. Each scenario generates its own local keys, package, and install root under a -per-scenario temp directory, so they are independent and use generated local -keys only — project-owned production signing keys remain an owner-controlled -release step. +per-scenario temp directory, so they are independent and sign with generated +local keys only. No scenario holds the production private key, which is an +owner-controlled release secret; lifecycle-03 and lifecycle-36 (Linux) and +lifecycle-17 (Windows) use the pinned production public key only to show that it +rejects a locally signed archive. If the host does not have `libcap-dev` or `libssl-dev`, run `bash scripts/setup-wsl-portable-build-deps.sh` first: it downloads the Ubuntu @@ -3712,6 +4318,23 @@ cargo test -p rocmd restart_runs_supervise_on_the_rocmd_it_is_given cargo test -p rocmd restart_without_a_rocmd_to_run_changes_nothing cargo test -p rocmd supervise_runs_the_engine_on_the_rocm_beside_rocmd cargo test -p rocmd supervise_without_a_rocm_beside_rocmd_changes_nothing +cargo test -p rocmd supervise_terminates_the_engine_when_recording_its_pid_fails +cargo test -p rocmd supervise_terminates_the_engine_when_recording_ready_fails +cargo test -p rocmd supervise_empties_a_log_left_by_an_earlier_run +cargo test -p rocmd an_engine_child_dropped_before_its_wait_is_terminated_and_reaped +cargo test -p rocmd an_engine_child_that_was_waited_for_is_not_signalled_on_drop +cargo test -p rocmd terminate_recovery_supervisor_kills_and_reaps_the_child_and_its_tree +cargo test -p rocmd a_termination_that_leaves_the_child_alive_neither_blocks_nor_stays_silent +cargo test -p rocmd a_termination_that_stops_the_child_reaps_it_and_reports_nothing +cargo test -p rocmd a_termination_with_unconfirmed_descendants_reaps_the_child_and_says_so +cargo test -p rocmd an_unconfirmed_termination_still_reaps_a_child_that_already_exited +cargo test -p rocmd a_failed_wait_leaves_the_engine_guard_armed +cargo test -p rocmd a_dropped_engine_guard_logs_a_termination_it_could_not_confirm +cargo test -p rocmd a_dropped_engine_guard_reports_to_stderr_by_default +cargo test -p rocmd a_report_that_cannot_be_written_does_not_panic +cargo test -p rocmd restart_reports_a_supervisor_it_could_not_confirm_stopped_in_its_error +cargo test -p rocmd restart_hands_the_supervisor_its_service_log_as_stderr +cargo test -p rocmd supervise_engine_output_does_not_overwrite_lines_appended_to_its_log cargo test -p rocmd engine_request_runs_the_stdio_command_on_the_rocm_beside_rocmd cargo test -p rocmd engine_request_without_a_rocm_beside_rocmd_names_it cargo test -p rocm-core sibling_binary_path_from @@ -3948,7 +4571,8 @@ to make the host match: masking every device with `HIP_VISIBLE_DEVICES=` makes the harness and the product agree the host has no usable GPU, which is what the mock lane has.) Note also that `-n` matches the scenario *name* (`serve-28 - Reuse succeeds on a GPU-less host`), not the `@id:` tag, so passing an id to it -silently selects nothing. +selects nothing and the run fails with `0 scenarios executed` (see +[E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-28, `@id:serve-reuse-exempt-from-no-gpu-refusal` — a `serve` that matches an already-running vLLM service on a GPU-less host is not refused: it @@ -4096,9 +4720,11 @@ whether a scenario may run at all (see the comment at a **prebuilt** `ROCM_CLI_BINARY`: the hook is compiled out there, the `@requires-oom-fault-injection` skip no longer applies, and the scenario fails for the wrong reason instead of being reported as skipped. Do not select it by -name on that route. Note also that `-n` matches the scenario *name* +name on that route, nor any other `@requires-oom-fault-injection` scenario +(`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so -passing an id to it silently selects nothing. +passing an id to it selects nothing and the run fails with +`0 scenarios executed` (see [E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-41, `@id:serve-oom-launch-memory-guidance` — a launch that runs out of GPU memory blames this launch, names the knobs, and quotes the engine's own @@ -4107,14 +4733,16 @@ passing an id to it silently selects nothing. ### The OOM fault-injection capability -serve-41 needs a managed launch that really failed on VRAM, which a GPU-less CI +The `@id:serve-oom-launch-memory-guidance` scenario needs a managed launch that really failed on VRAM, which a GPU-less CI host cannot produce. It is therefore driven by a test-only fault-injection hook: the `rocm` crate feature `e2e-oom-fault-injection`, armed at run time by `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`, makes `rocm serve` fabricate the on-disk state of a managed vLLM launch that spawned, wrote an allocator OOM traceback to the -log it owns, and never became ready. No process is spawned, and the seam -compiles out entirely without the feature, so a build that does not enable it -behaves exactly as the shipped binary does. +log it owns, and never became ready. `@id:serve-smoke-failure-summary` arms the same hook with +`ROCM_E2E_SIMULATE_OOM_LAUNCH=ready`, which fabricates a launch that did become +ready, so a mock engine can then fail its smoke test. No process is spawned, and +the seam compiles out entirely without the feature, so a build that does not +enable it behaves exactly as the shipped binary does. Only `cargo xtask e2e` enables it, by building the binary under test with `--features "rocm/e2e-test-hooks rocm/e2e-oom-fault-injection"`, and it is the @@ -4127,7 +4755,7 @@ was handed a prebuilt binary. Consequently, every lane that runs a **prebuilt** `ROCM_CLI_BINARY` — the self-hosted lanes in [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml) and the [`nightly.yml`](../.github/workflows/nightly.yml) lanes — has the hook -compiled out and resolves serve-41 to a reported **skip**, not a silent pass. It +compiled out and resolves both of those scenarios to a reported **skip**, not a silent pass. It gates PRs on the GitHub-hosted mock lane, which lets `cargo xtask e2e` build the binary itself. The prebuilt lanes' `--features` floor is asserted as a superset check rather than an exact match for this reason; see the test notes on @@ -4137,11 +4765,99 @@ check rather than an exact match for this reason; see the test notes on Be precise about *which* gate produces the skip, because the hook is not always the one that decides. On the Windows lifecycle lane in [`ci.yml`](../.github/workflows/ci.yml) the hook is compiled out too, but -serve-41 is never selected there at all: that lane runs `E2E_ONLY_LIFECYCLE` and -the scenario carries `@requires-os:linux`. On the GPU lanes it is +both scenarios are never selected there at all: that lane runs +`E2E_ONLY_LIFECYCLE` and the scenarios carry `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. +## Serve Smoke Test Failure + +After a fresh launch becomes ready, the `rocm serve` interactive summary sends +one small chat request. When that request cannot complete the summary is headed +`Deployment summary (smoke test failed)`, shows the status as +`started, smoke test failed` instead of `ready`, and a hedged note gives the +reason and names `rocm logs --service `. See +[the README](../README.md#model-serving) for the user-facing contract. The +launch itself stays best-effort: exit code and service record do not change, and +`--no-smoke-test` and an already-running service are unchanged. + +What is detected, and what is not: + +- Detected (the summary says the smoke test failed): a transport error or timeout, + an HTTP status other than 200, a parse error, and a chunked or `Content-Length` + stream cut short. +- Not detected (no failure is reported and the status reads `ready`): an engine + error sent as a `data:` payload inside a `200` stream; a `200` answer with no + `data:` line at all (HTML, plain JSON, an empty body); a close-delimited stream + cut after a token; and a complete stream with no `content` token, which is + deliberately not a failure (a reasoning model can spend its whole small budget + first, as the readiness probe also allows for). The metric rows read `n/a` when + no content token arrived first; tokens that did arrive (the close-delimited cut, + or an in-band `data:` error after content) are measured. +- Known limitation: the reason shown is whatever the provider call reported, and + some of its errors are not the engine's, for example the `/v1/models` re-list + before the request ("no ready managed service"), an unreadable services-directory + entry, a rejected endpoint URL, and the audit event written after the stream is + consumed. Each turns into a failed smoke test, which is why the note says the + smoke test could not complete and only that the server may not be able to serve + requests. +- Known limitation: the smoke request is routed by model name, not to the launched + service. The newest ready service serving the model is used, so normally the one + just launched. If the new service fails the 2 s `/v1/models` listing probe (for + example the engine was killed just as it became ready) it is filtered out: an + older ready service with the same model then answers, and the summary shows + `ready` with that service's metrics and no note, a false pass on a dead launch. + With no older service the reason shown is rocm's "no ready managed service", not + the engine's. + +Focused unit coverage: + +```bash +cargo test -p rocm --bin rocm serve_smoke_test +cargo test -p rocm --bin rocm a_failed_smoke_test_is_not_presented_as_ready_and_says_why +cargo test -p rocm --bin rocm a_passing_or_skipped_smoke_test_adds_no_failure_note +cargo test -p rocm --bin rocm an_already_running_summary_ignores_a_smoke_failure +cargo test -p rocm --bin rocm failure_reason_ +cargo test -p rocm --bin rocm the_commands_the_failure_note_names_exist_and_parse +``` + +The `serve_smoke_test` tests in [providers.rs](../apps/rocm/src/providers.rs) +run the real smoke test against a local mock engine over a socket (an engine that +dies mid-answer or inside a chunk, a 500, 401 and 404, a hostile status line, a +complete stream with no content, a close-delimited stream cut after a token). Only +the mid-stream-death test goes on to render the summary from the real outcome; the +others assert on the outcome the smoke test returns. + +- `@id:serve-smoke-failure-summary` (serve-60) — a launch that becomes ready + whose engine fails its first chat request is shown as a failed smoke test, with + the engine's own reason and the full `rocm logs --service ` command, and + the engine received exactly one chat request. It carries `@requires-no-gpu`, + `@requires-os:linux` and `@requires-oom-fault-injection`: it uses the same + test-only hook as `@id:serve-oom-launch-memory-guidance` (see [Serve OOM Guidance](#serve-oom-guidance)) in its + `ready` mode, a ready record pointing at a mock engine whose chat requests all + fail with HTTP 500, so it has that scenario's gates: against a prebuilt + `ROCM_CLI_BINARY` the hook is compiled out and the harness reports it as a skip, + unless it is selected with `-n` or `--tags`, which bypasses that skip and makes + it fail for the wrong reason. Some of its steps are not checks of the fix, so say + plainly what they prove: + - The exit code 0 is asserted in the `When` step (`wait_for_exit`), not in a + `Then` step. + - "the engine received exactly one chat request" passes with the fix reverted + (the smoke request was already sent); it shows the request was not retried. + - "the engine's own reason" matches rocm's wording of the status line around the + engine's HTTP status, so it pins that phrasing as well. + - "the launch's service record is left as ready" passes with the fix reverted; + it is the only assertion behind the "ROCm CLI does not stop the server" + promise in the README and `rocm serve --help`. + - "no background automation daemon was left behind" is hygiene only: it records + that no daemon was started but does not discriminate the + `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs`, because no watcher is + enabled in the isolated config and the helper starts only when one is. + +```bash +cargo xtask e2e -- -n "serve-60" +``` + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global @@ -4391,7 +5107,7 @@ the probe finds. `examine-34` plants the same machine with `libhsa-runtime64.so.1` beside the library and `LD_LIBRARY_PATH` empty: the runtime opens `librocdxg.so` through its -own `$ORIGIN` runpath, so `driver_status` is `wsl_rocdxg_ready` with neither the +own `$ORIGIN` search path, so `driver_status` is `wsl_rocdxg_ready` with neither the cache nor the variable involved, which is the layout of a TheRock tarball (ROCM-30386). Its control is `examine-33`, which plants the library alone and stays `wsl_rocdxg_missing`, so the sibling runtime, and not the planted library, @@ -4409,6 +5125,20 @@ RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-3[5-8] ' RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-48 ' ``` +`examine-49` to `examine-53` cover the managed-runtime `librocdxg` finding on WSL (see [wsl.md](wsl.md)). `examine-49` to `examine-51` and `examine-53` plant the WSL machine of `examine-32` (`examine-52` plants a host that is not WSL) plus a managed wheel runtime in the registry whose library directory holds `libhsa-runtime64.so.1` and `librocdxg.so.1`. + +`examine-49` has no `librocdxg.so` beside them, so `rocm examine --json` lists that `librocdxg.so.1` in `summary.unlinked_runtime_librocdxg` while `driver_status` stays `wsl_rocdxg_ready`. `examine-50` is the control: the same runtime with `librocdxg.so` present, and nothing is listed. `examine-51` reads the text form, runs the `ln -sfn` command its `wsl_runtime_librocdxg` line advises, and inspects again, so the advice is shown to exist, to clear the finding and to make `librocdxg.so` read as the runtime's own copy. `examine-53` plants a dangling `librocdxg.so` (a link to a missing file) beside the runtime's copy: the inspection lists it, the user runs the advised command (`ln -sfn`, since a plain `ln -s` fails with "File exists" on a dangling link), and a second inspection lists nothing and `librocdxg.so` reads as the runtime's own bytes. `examine-52` is the same runtime on a machine that is not WSL, where nothing is listed. Each run sets `ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK`, because a registered runtime would otherwise start the update check, which needs the network and prints ahead of the JSON. + +The install side (`rocm install sdk` linking the name on WSL) has no scenario, because it needs a real wheel download; the helper unit tests in [runtime_rocdxg.rs](../crates/rocm-core/src/runtime_rocdxg.rs) cover it against a temporary directory: the link is made and resolves to the runtime's copy, nothing happens off WSL, a refused link is reported with its error and the manual command, a dangling `librocdxg.so` is replaced atomically (no temporary link left; also after an injected rename failure, where removal is the real one), and a name that vanished between the check and the re-check is linked, and a `librocdxg.so` that resolves when it is checked, a file or a working link, is left alone (the name is re-checked before the rename: still dangling or gone, the temporary is renamed into place; resolving by then, it is refused. A name that becomes a working link between the re-check and the rename is replaced; the re-check narrows that window and cannot close it, and its length is not measured). The versioned copy decides whether a directory is listed or linked at all: a file or a resolving link is, and an absent, dangling or looping one is not, through examine's scan as well as the installer. That the report links only the probe's library directories is asserted by `the_install_report_links_only_the_probes_library_directories_when_wsl` in [therock.rs](../apps/rocm/src/therock.rs), not by those helper tests, which take a bare directory list. + +The entry point the install calls reads the WSL gate itself, and `the_install_entry_point_reads_the_wsl_gate_from_the_host` drives it on a simulated WSL host and a simulated native one through `ROCM_CLI_TEST_HOST_ROOT`; it runs only with the hooks feature, so run it with `cargo test -p rocm-core --lib --features e2e-test-hooks -- runtime_rocdxg`. + +Deleting the call in `install_wheel_runtime` is caught by the dead-code lint under `cargo clippy --workspace --all-targets -- -D warnings` (the required lint job), not by a test: with the call removed, clippy reports `function report_runtime_librocdxg_link is never used` (and the two helpers only it uses), while `cargo test -p rocm --bin rocm therock::` stays green with one warning. A wrong gate or directory list passed through the one-line wrapper is covered by the tests of the shared helpers (`the_install_report_links_only_the_probes_library_directories_when_wsl`, driven with the gate true and false on every Unix host, and the hooks-lane test of the real gate), but a different list passed to the core entry from inside the wrapper is not caught by any test. A possible follow-up, not done here: the gated `runtime-01` scenario (`@requires-gpu @nightly`) could assert the `librocdxg.so` link after its real install when it runs on WSL. The scenarios run on the Linux lanes and need no GPU. + +```bash +RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-(49|50|51|52|53) ' +``` + `serve-42` plants the same WSL machine with `LD_LIBRARY_PATH` empty and runs `rocm serve`: it is refused, its advice names the WSL `driver_status` and not `/dev/kfd`, and `rocm examine` on that machine reports `wsl_rocdxg_missing`, so the command the refusal names exists and shows the condition. The launch that follows when `LD_LIBRARY_PATH` does reach the library has no scenario, because it would start preparing a real engine; the serve gate's decision there is covered by unit tests in `gpu_visibility.rs`. ```bash @@ -4697,9 +5427,9 @@ hold the behaviour end to end, GPU or not: runs only the bash half, on Linux, and ci.yml's Windows lane does not run this scenario. - `diagnose-60` (Linux and WSL lanes only, like every pseudo-terminal - scenario here; the Windows key is checked by hand in - [manual-testing.md](manual-testing.md) section 13) runs - `rocm diagnose --symptom-file -` on a pseudo-terminal tall enough to keep + scenario here; the Windows key is checked by hand in the "Pasting An Error + Into `rocm diagnose`" section of [manual-testing.md](manual-testing.md)) + runs `rocm diagnose --symptom-file -` on a pseudo-terminal tall enough to keep the whole exchange on screen (it has no scrollback), requires the hint telling the user how to end the paste, then types the paste and Ctrl-D, the Linux end-of-input key the hint names, and requires the screen, which also diff --git a/docs/ux-guidelines.md b/docs/ux-guidelines.md index eb71fb298d..72b161daee 100644 --- a/docs/ux-guidelines.md +++ b/docs/ux-guidelines.md @@ -179,3 +179,11 @@ Command navigability follow-up notes: - Keep backend labels such as wheels, tarballs, runtime keys, registries, and daemon policy out of first-visible TUI screens unless the user opens an explicit advanced/debug view. + +## Machine-Readable Output + +- Text for a person is the default; `--json` is an explicit opt-in per command. +- With `--json`, stdout carries one JSON document and nothing else. Progress, + warnings and diagnostics go to stderr so a piped document stays parseable. +- Which commands have `--json`, the frozen contracts, and the rules for + changing a documented field are in [json-output.md](json-output.md). diff --git a/docs/vllm.md b/docs/vllm.md index 47eaa53f86..f386417aec 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -109,15 +109,33 @@ SDK and torch wheels are published for 3.10 to 3.14, and `rocm install sdk vLLM install then refuses that runtime; re-running the install with `--python-version 3.14` moves it back. -This route is selected by `major.minor`, so `10.0.0` and `10.1.0` discover -through *different* rows: they pin different vLLM minors and read vLLM from -different index URLs, because AMD stages ROCm 10.1's vLLM builds on a separate -host from 10.0's production index. Both rows take their torch stack from the -same `whl-next` index, which is also where `rocm install sdk` resolves either -SDK's own `+rocmX.Y` torch from. vLLM's discovery is independent of that: it -resolves its own torch, torchvision, and torchaudio pins fresh from the row's -own static version prefixes, instead of reusing what the SDK install resolved. -The SDK install might have resolved a different torch version than the row pins. +This route is selected by `major.minor`, so `10.0.0`, `10.1.0` and `10.2.0` +discover through *different* rows, each pinning its own vLLM minor. `10.0.0` reads +vLLM from the production index; `10.1.0` and `10.2.0` read it from a separate +staging host, where AMD stages their vLLM builds. The `10.0.0` and `10.1.0` rows +take their torch stack from the stable `whl-next` index, which is also where +`rocm install sdk` resolves those SDKs' own `+rocmX.Y` torch from. For those two +rows vLLM's discovery is independent of the SDK install: it resolves its own +torch, torchvision, and torchaudio pins fresh from the row's own static version +prefixes, instead of reusing what the SDK install resolved. The SDK install might +have resolved a different torch version than the row pins. + +The `10.2.0` row does not discover its torch stack: it keeps the torch, +torchvision and torchaudio the SDK install already resolved (`torch 2.14.*`, +torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them +against the row's series and the SDK's ROCm line, and realigns to those exact +builds from the index the SDK was installed from (the one its runtime manifest +records, so a nightly SDK uses the nightly index and a release SDK the release +one; the row's nightly index when no manifest for the SDK's interpreter recorded +one or the registry could not be read, and the install output says which). A series match +would not work there: the nightly index serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a +`+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it +requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that +staging does not serve. The row has been installed and imported (`import +vllm._C`) against a 10.2 SDK. A GPU serve has been run on WSL2 only with the +runtime linked to its own `librocdxg.so`, never against a release SDK, and the 10.2 vLLM +wheel is published for `linux_x86_64` only. The nightly channel resolves to +ROCm 10.3, which has no vLLM build and no row. One value flows the other way. A row's exact torchaudio pin, where it has one instead of a series, is also the hardware-verified build `rocm install sdk` pins @@ -140,6 +158,49 @@ cannot silently install the wrong line's wheel onto this SDK. If AMD's index has no compatible build for a package, the resolver fails and the install fails instead of falling back to an unpinned or CPU install. +One package is checked after the install instead. vLLM's wheel pins +`tensorizer==2.10.1`, which cannot be imported under Python 3.14 +(`KeyError: 'crypto_stream_salsa20_NONCEBYTES'`), and ROCm 10.1.0's vLLM is +published for Python 3.14 only. The install, and every later +`rocm engines install vllm` on a managed runtime, imports +`tensorizer.serialization` with the runtime's Python, under the environment the +server is launched with (including `PYTHONPATH`) and with colour switched off. +When that import fails inside tensorizer itself, it upgrades tensorizer to +2.12.1 with `--no-deps` and imports it again. A torch or shared-library failure +is not tensorizer's and is left to the device check. + +When this run installed vLLM (a first install, `--reinstall` or a repair), a +failed upgrade, a check that cannot run, or a tensorizer that still cannot be +imported after the upgrade fails the install with the exception, instead of +ending in a startup crash at `rocm serve`. On a runtime this run did not +install vLLM into, the same three cases are warnings, so an install that used to +be a no-op (for example offline) stays one. A tensorizer that imports keeps +vLLM's pin. A later `rocm engines install vllm` on a runtime whose upgrade +failed only warns, and `rocm serve` on it keeps failing until an install +succeeds: retrying after a failed install is deliberately not fatal. + +The check runs inside the engine install, before the CLI realigns the runtime's +torch. If torch cannot load against the SDK at that moment, the probe sees a +torch failure rather than tensorizer's, reads it as not tensorizer's and leaves +tensorizer alone; the next `rocm engines install vllm` checks again. + +Because the upgraded release no longer matches vLLM's exact pin, +`uv pip check` reports it, and `dependency_check` shows it as +`expected_divergence` (not `violated`), with a note naming the tensorizer release +the runtime holds. A reinstall is not needed to keep it: the install re-checks +tensorizer and upgrades it again, provided torch can load when the check runs +(see above). + +Two side effects are new. The check imports torch and tensorizer under the +environment the server is launched with, which can write the launch's +compatibility shims, so a `rocm engines install vllm` on an existing managed +runtime is no longer free of side effects. The probe and the upgrade each have a +five-minute timeout. And on the `rocm install sdk` auto-install path, a refused +tensorizer upgrade can still print `dependency_check: satisfied` beside the +refusal error, because a refused upgrade keeps vLLM's own pin and `uv pip check` +only compares declared metadata; the refusal text carries the truth, and that +output is deliberately not changed here. + `flash-attn` and `amd-aiter` are the exception to the per-package series match above. The vLLM wheel declares the exact builds of both that it was built against, and the index keeps publishing newer ones it does not accept (for diff --git a/docs/wsl.md b/docs/wsl.md index 197b51747b..b728ec8996 100644 --- a/docs/wsl.md +++ b/docs/wsl.md @@ -14,7 +14,10 @@ virtual environments and ROCDXG (`librocdxg`). The AMD WSL path is: 1. Windows 11 with the AMD Adrenalin WSL-capable driver. -2. WSL2 with Ubuntu 24.04 or newer. +2. WSL2 with Ubuntu 24.04 or newer. Ubuntu 22.04 is not supported: it ships + glibc 2.35, below what the Lemonade embeddable ROCm CLI pins requires + (glibc 2.38 and `GLIBCXX_3.4.32`; newer Lemonade releases need more), so + the Lemonade engine cannot start there. Ubuntu 24.04 provides glibc 2.39 and `GLIBCXX_3.4.33`. 3. ROCDXG (`librocdxg`) installed inside WSL. 4. A TheRock runtime installed by ROCm CLI into a managed Python venv. @@ -317,11 +320,43 @@ below. entry - whether `librocdxg.so` is in the same `lib` directory as a `libhsa-runtime64` (`wsl_librocdxg_beside_runtime`). The runtime opens `librocdxg.so` by name and - carries `RUNPATH $ORIGIN`, so a TheRock tarball under `/opt/rocm-`, + searches its own directory (`$ORIGIN`; the wheel runtimes' `libhsa-runtime64.so.1` + carries it as a `DT_RPATH`, per `readelf -d`), so a TheRock tarball under `/opt/rocm-`, which keeps the two side by side and registers nothing with `ldconfig`, loads with neither the cache nor the variable. `driver_status` is `wsl_rocdxg_ready` when any of the three sources finds the library, and `wsl_rocdxg_missing` only when none does +- a managed wheel runtime whose own `librocdxg.so.1` its runtime does not load + (`wsl_runtime_librocdxg` in the text form, `summary.unlinked_runtime_librocdxg` + in `--json`; printed only when there is one). The wheel ships + `librocdxg.so.1` beside `libhsa-runtime64`, but the runtime opens + `librocdxg.so`, which the wheel does not ship (or which is a link to a file + that is missing: the loader's `open` fails on a broken link as on an absent + file), so it loads the host's copy. + A host `rocdxg-roct` older than the runtime expects then fails at first GPU + use, for example ROCm 10.2 and 10.3 nightly runtimes against `rocdxg-roct` + 1.2.2 (`undefined symbol: hsaKmtSetPersistingCacheSize`, then a segmentation + fault), while `driver_status` still reads `wsl_rocdxg_ready`, which describes + the host's plumbing. Wheel runtime installs and updates (`rocm install sdk`, `rocm update --apply`) + link `librocdxg.so` to the runtime's own copy on WSL (tarballs ship both + names, and an adopted runtime is never linked); for a runtime installed earlier the line names the + `ln -sfn librocdxg.so.1 /librocdxg.so` command that does the same + (`-fn` so that it also replaces a dangling `librocdxg.so`; the install replaces + one atomically, and leaves alone a `librocdxg.so` that resolves when it is + checked; the install refuses at its final check if the name resolves by then, + and a name that becomes a working link between that check and the rename is + replaced). The + versioned name is the fixed `librocdxg.so.1`: a wheel whose soname changes is + not detected or linked, and a dangling `librocdxg.so` beside a runtime with no + `librocdxg.so.1` is not detected. A `librocdxg.so.1` that is itself dangling or + a link loop is not listed or linked either, and is deliberately not reported + (nothing valid to link to). + The mechanism (the unversioned open skipping the runtime's directory and + landing on `/opt/rocm/lib`) was observed with `LD_DEBUG=libs` on the 10.2 + nightly; which component requests `hsaKmtSetPersistingCacheSize` is not + identified. The line is a pointer, not a verdict: it appears for every such runtime, + including ones that work with the host's copy, because the files alone do not + say which symbols the runtime needs - the Windows host AMD driver version, when WSL interop can reach the host (`--json` only) - whether `HSA_ENABLE_DXG_DETECTION` is set (`--json` only) diff --git a/doctor-catalog.json b/doctor-catalog.json index 64580f82b4..82ee7bf66d 100644 --- a/doctor-catalog.json +++ b/doctor-catalog.json @@ -456,7 +456,7 @@ }, "exit_code_descriptions": { "declined": "The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.", - "failed": "A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.", + "failed": "A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.", "internal": "The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.", "not_applicable": "The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).", "ok": "Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.", diff --git a/engines/aim/Cargo.toml b/engines/aim/Cargo.toml index 97b7d0265d..4454f2528e 100644 --- a/engines/aim/Cargo.toml +++ b/engines/aim/Cargo.toml @@ -26,3 +26,7 @@ serde.workspace = true serde_json.workspace = true tempfile = "3" ureq = { version = "2.12", features = ["json", "native-certs"] } + +[dev-dependencies] +# The shared engine protocol contract check (`engine_contract`). +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } diff --git a/engines/aim/src/lib.rs b/engines/aim/src/lib.rs index 3bab19210d..a3deca7b2f 100644 --- a/engines/aim/src/lib.rs +++ b/engines/aim/src/lib.rs @@ -155,11 +155,7 @@ pub fn run_cli() -> Result<()> { engine_recipe: None, gpu_selection: crate::process::parse_gpu_selection_arg(gpu.as_deref())?, })?), - CommandKind::Stdio => { - let envelope = read_request()?; - let response = handle_envelope(envelope); - print_json(&response) - } + CommandKind::Stdio => print_json(&answer_stdio(&read_request()?)), CommandKind::ServeHttp { service_id, model_ref, @@ -229,6 +225,11 @@ pub fn builtin_serve_http( } fn handle_envelope(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope { + // Refuse a request made for a protocol this engine cannot serve before its + // payload is read: in another protocol the payload may mean something else. + if let Some(refusal) = rocm_engine_protocol::incompatible_request_response(&envelope) { + return refusal; + } match envelope.method { EngineMethod::Detect => { deserialize_and_respond::(envelope.payload, |request| { @@ -269,7 +270,10 @@ where let request: T = match serde_json::from_value(payload) { Ok(request) => request, Err(error) => { - return EngineResponseEnvelope::failure("invalid_payload", error.to_string()); + return EngineResponseEnvelope::failure( + rocm_engine_protocol::INVALID_PAYLOAD_CODE, + error.to_string(), + ); } }; match handler(request) { @@ -413,12 +417,21 @@ pub(crate) const fn windows_gpu_unsupported_message() -> &'static str { serving is skipped." } -fn read_request() -> Result { +/// The request text on stdin, read to its end. It is decoded by +/// [`answer_stdio`], which judges its protocol version first. +fn read_request() -> Result { let mut input = String::new(); std::io::stdin() .read_to_string(&mut input) .context("failed to read engine stdio request")?; - serde_json::from_str(&input).context("failed to parse engine stdio request") + Ok(input) +} + +/// The answer to the request `text` read on stdin: its protocol version is +/// judged before its method or payload is decoded (see +/// [`rocm_engine_protocol::answer_engine_request`]), then it is handled. +fn answer_stdio(text: &str) -> EngineResponseEnvelope { + rocm_engine_protocol::answer_engine_request(text, handle_envelope) } fn print_json(value: &T) -> Result<()> { @@ -428,3 +441,19 @@ fn print_json(value: &T) -> Result<()> { writeln!(handle)?; Ok(()) } + +#[cfg(test)] +mod protocol_tests { + use super::*; + + /// The engine protocol contract, the same check every engine runs: see + /// [`rocm_engine_protocol::engine_contract`]. + #[test] + fn stdio_answers_hold_the_engine_protocol_contract() { + rocm_engine_protocol::engine_contract::assert_engine_holds_protocol_contract( + "aim", + answer_stdio, + handle_envelope, + ); + } +} diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index 19c7f652ec..c81710afbd 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -179,6 +179,15 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } +/// Serializes `value` to `path` as pretty JSON, creating the parent directory +/// if needed. The low-level primitive [`write_running_state`], +/// [`merge_state_field`] and [`write_terminal_state`] all go through — it is +/// not the only way this module's state files are written: +/// inference-verification latches go through +/// `rocm_core::merge_json_state_file` instead (see [`inference_verified`]), +/// which patches the file under the state directory's lock rather than +/// routing through here. This function takes no such lock and writes in +/// place, so the lock does not order it against those latches. fn write_state(path: &Path, value: &Value) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) diff --git a/engines/aim/tests/stdio_binary.rs b/engines/aim/tests/stdio_binary.rs new file mode 100644 index 0000000000..95b94ffd7f --- /dev/null +++ b/engines/aim/tests/stdio_binary.rs @@ -0,0 +1,67 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `rocm-engine-aim stdio` binary, end to end: what it reads on stdin goes +//! through the shared request reader, so a request it cannot serve, or input +//! that is not a request, is answered with an error envelope on stdout and a +//! clean exit. + +use std::io::Write as _; +use std::process::{Command, Stdio}; + +/// Run `rocm-engine-aim stdio` with `request` on stdin, on scratch app roots, +/// and return its exit status and the error code of the envelope it printed. +fn answer(label: &str, request: &str) -> (Option, String) { + let root = std::env::temp_dir().join(format!( + "rocm-engine-aim-stdio-{label}-{}", + std::process::id() + )); + let mut child = Command::new(env!("CARGO_BIN_EXE_rocm-engine-aim")) + .arg("stdio") + .env("ROCM_CLI_CONFIG_DIR", root.join("config")) + .env("ROCM_CLI_DATA_DIR", root.join("data")) + .env("ROCM_CLI_CACHE_DIR", root.join("cache")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start the engine binary"); + child + .stdin + .take() + .expect("piped stdin") + .write_all(request.as_bytes()) + .expect("write the request"); + let output = child + .wait_with_output() + .expect("wait for the engine binary"); + let _ = std::fs::remove_dir_all(&root); + let stdout = String::from_utf8_lossy(&output.stdout); + let envelope: serde_json::Value = serde_json::from_str(&stdout).unwrap_or_else(|error| { + panic!( + "{label}: no envelope on stdout ({error}): {stdout:?}; stderr: {}", + String::from_utf8_lossy(&output.stderr) + ) + }); + let code = envelope["error"]["code"] + .as_str() + .unwrap_or_default() + .to_owned(); + (output.status.code(), code) +} + +#[test] +fn the_stdio_binary_answers_requests_it_cannot_serve() { + assert_eq!( + answer( + "incompatible", + r#"{"protocol_version":"9.0.0","method":"frobnicate","payload":{}}"# + ), + (Some(0), "incompatible_protocol".to_owned()) + ); + assert_eq!( + answer("garbage", "this is not a request"), + (Some(0), "invalid_request".to_owned()) + ); +} diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index 98b56d0681..53ec95e49c 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -37,7 +37,17 @@ ureq = { version = "2.12", features = ["json", "native-certs"] } # extractor's dependency surface small. zip = { version = "9.0", default-features = false, features = ["deflate"] } +[target.'cfg(target_os = "linux")'.dependencies] +# Reads the version nodes the host's libc.so.6 and libssl.so.3 define, to +# refuse a host older than the pinned `lemond` needs (`src/host_libraries.rs`). +# 0.37 with only the ELF reader: `backtrace` already links this version into +# `rocm`, so it adds no crate there. The standalone `rocm-engine-lemonade` +# binary has no `backtrace`, so for it `object` is one new crate. +object = { version = "0.37", default-features = false, features = ["read_core", "elf"] } + [dev-dependencies] +# The shared engine protocol contract check (`engine_contract`). +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # Archive-extraction property tests; see `crates/rocm-core/src/archive_props.rs`. proptest = { version = "1", default-features = false, features = ["std"] } diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index dc07423f1d..b7ccd91f2c 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -94,7 +94,7 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( disabled, try_llamacpp_backend_alignment, install_best_llamacpp_backend, - latest_llamacpp_rocm_stable_tag, + newest_llamacpp_rocm_stable_tag_for, ) }, ) @@ -340,7 +340,7 @@ fn install_pinned_backend_or_restore_packaged_pins( /// The Tier 1 / Tier 2 / revert state machine, isolated from the config and active-SDK /// lookups above so it can be exercised in tests against a temp `backend_versions.json` -/// with the install/align/latest-tag steps injected, instead of spawning a real +/// with the install/align/tag-lookup steps injected, instead of spawning a real /// `lemond` and reaching GitHub. /// /// `disabled` is the resolved value of [`lemonade_backend_alignment_disabled`], passed @@ -364,7 +364,7 @@ fn align_llamacpp_backend_to_version( disabled: bool, mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, - mut latest_tag: impl FnMut() -> Result, + mut newest_tag_with_asset: impl FnMut(&str) -> Result, ) -> Result> { if disabled { fallback_install(manifest, false)?; @@ -392,31 +392,41 @@ fn align_llamacpp_backend_to_version( // matching ROCm-version asset — Lemonade only started publishing per-ROCm- // version builds partway through its release history, and the embeddable's // pin (frozen at whatever build was current when this Lemonade version was - // cut) does not track that. Point at the current newest build instead, still - // paired with the active ROCm version, forcing a real reinstall attempt - // regardless of whatever Tier 1 left on disk. + // cut) does not track that. Point at the newest build that actually ships an + // archive for the active ROCm version instead, still paired with that version, + // forcing a real reinstall attempt regardless of whatever Tier 1 left on disk. let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag_applied = match latest_tag() { - Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { - match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { + let found_tag_applied = match newest_tag_with_asset(target_version) { + Ok(tag) if pinned_tag.as_deref() != Some(tag.as_str()) => { + match write_backend_versions_llamacpp_tag(backend_versions_path, &tag) { Ok(()) => true, Err(error) => { eprintln!( - "Warning: could not pin Lemonade's backend to llama.cpp {latest_tag}: \ + "Warning: could not pin Lemonade's backend to llama.cpp {tag}: \ {error:#}" ); false } } } - // The pinned tag already is the latest — Tier 1 already tried it. + // The pinned tag already is that build — Tier 1 already tried it. Ok(_) => false, Err(error) => { - eprintln!("Warning: could not determine Lemonade's latest llama.cpp build: {error:#}"); + eprintln!( + "Warning: could not look up a llama.cpp build that ships an archive for ROCm \ + {target_version}: {error:#}" + ); false } }; - if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { + if found_tag_applied + && align( + manifest, + target_version, + true, + "the newest llama.cpp build for this ROCm version", + ) + { return Ok(Some(target_version.to_owned())); } @@ -439,7 +449,7 @@ fn align_llamacpp_backend_to_version( ({pinned_version}): {error:#}" ); } - if latest_tag_applied { + if found_tag_applied { // Best-effort, matching the therock.version restore above: a failure here // must not skip the fallback reinstall below. Restore the original tag when // there was one; otherwise Tier 2 pinned a key that did not exist before, so @@ -544,36 +554,117 @@ const fn llamacpp_backend_alignment_succeeded( /// `resources/backend_versions.json`'s `therock.version` alone doesn't get a /// ROCm-version-matched build if the pinned tag predates that ROCm version's /// support: Lemonade only started publishing per-ROCm-version assets partway -/// through this repo's release history, and the embeddable's own pin can be -/// (and, at the time of writing, is) older than that. +/// through this repo's release history, and an embeddable's own pin can be older +/// than that (Lemonade 11.5.1's b9752 was). const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; -/// The newest published release tag for [`LLAMACPP_ROCM_STABLE_REPO`] (e.g. -/// `b10952`), queried directly rather than assumed — the whole reason Tier 2 -/// exists is that the embeddable's own pinned tag cannot be trusted to have a -/// matching-ROCm-version asset. -fn latest_llamacpp_rocm_stable_tag() -> Result { - let url = format!("https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases/latest"); - let timeout = Duration::from_secs(15); +/// How many of [`LLAMACPP_ROCM_STABLE_REPO`]'s newest releases Tier 2 searches +/// for one that ships the archive it needs: one page of GitHub's release listing, +/// so a single request. How far back that reaches depends on how often the +/// repository publishes, which this repo does not control. +const LLAMACPP_RELEASES_SEARCHED: usize = 30; + +/// The `llamacpp:rocm-stable` archive Lemonade downloads for build `tag` paired +/// with ROCm `rocm_version` — the name Lemonade's own installer builds, from the +/// ROCm version's `major.minor` (`10.1.0` -> `rocm-10.1`). +fn llamacpp_rocm_stable_asset_name(tag: &str, rocm_version: &str, windows: bool) -> String { + let major_minor = rocm_version + .splitn(3, '.') + .take(2) + .collect::>() + .join("."); + if windows { + format!("llama-{tag}-bin-win-rocm-{major_minor}-x64.zip") + } else { + format!("llama-{tag}-bin-ubuntu-rocm-{major_minor}-x64.tar.gz") + } +} + +/// From a GitHub release listing (`GET /repos/{repo}/releases`), the tag of the +/// most recently published release that actually ships the `llamacpp:rocm-stable` +/// archive for `rocm_version` on this platform. +/// +/// The newest release is not good enough on its own: a release can be published +/// before, or without, its ROCm archives (`b12013` through `b12086` carry only +/// qualification reports), and a build can skip one ROCm version or one platform. +/// Drafts, prereleases, and assets not reported as `uploaded` are skipped. +fn newest_llamacpp_tag_with_asset( + releases: &Value, + rocm_version: &str, + windows: bool, +) -> Option { + releases + .as_array()? + .iter() + .filter(|release| { + !release + .get("draft") + .and_then(Value::as_bool) + .unwrap_or(false) + && !release + .get("prerelease") + .and_then(Value::as_bool) + .unwrap_or(false) + }) + .filter_map(|release| { + let tag = release.get("tag_name")?.as_str()?; + let wanted = llamacpp_rocm_stable_asset_name(tag, rocm_version, windows); + let ships_asset = release.get("assets")?.as_array()?.iter().any(|asset| { + asset.get("name").and_then(Value::as_str) == Some(wanted.as_str()) + && asset.get("state").and_then(Value::as_str) == Some("uploaded") + }); + let published_at = release.get("published_at")?.as_str()?; + ships_asset.then_some((published_at, tag)) + }) + .max_by(|left, right| left.0.cmp(right.0)) + .map(|(_, tag)| tag.to_owned()) +} + +/// The newest [`LLAMACPP_ROCM_STABLE_REPO`] build that ships an archive for +/// `rocm_version` on this platform (e.g. `b12003`), queried directly rather than +/// assumed — the whole reason Tier 2 exists is that the embeddable's own pinned +/// tag cannot be trusted to have a matching-ROCm-version asset. +fn newest_llamacpp_rocm_stable_tag_for(rocm_version: &str) -> Result { + newest_llamacpp_rocm_stable_tag_via(rocm_version, fetch_github_json) +} + +/// GET `url` from the GitHub API and parse the body as JSON. +fn fetch_github_json(url: &str) -> Result { // `timeout_connect` takes precedence over `timeout` and defaults to 30s, so without // it a host that blackholes rather than refuses would stall well past the intended - // ceiling. + // ceiling. The overall timeout also covers reading the listing. let agent = ureq::AgentBuilder::new() - .timeout_connect(timeout) - .timeout(timeout) + .timeout_connect(Duration::from_secs(15)) + .timeout(Duration::from_secs(30)) .build(); let response = agent - .get(&url) + .get(url) .set("User-Agent", "rocm-cli") .call() .with_context(|| format!("failed to query {url}"))?; - let body: Value = response + response .into_json() - .with_context(|| format!("failed to parse response from {url}"))?; - body.get("tag_name") - .and_then(Value::as_str) - .map(str::to_owned) - .with_context(|| format!("{url} response is missing 'tag_name'")) + .with_context(|| format!("failed to parse response from {url}")) +} + +/// [`newest_llamacpp_rocm_stable_tag_for`] with the listing fetch injected, so +/// a test reaches the host-platform choice without the network. +fn newest_llamacpp_rocm_stable_tag_via( + rocm_version: &str, + fetch: impl FnOnce(&str) -> Result, +) -> Result { + let url = format!( + "https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases?per_page={LLAMACPP_RELEASES_SEARCHED}" + ); + let body = fetch(&url)?; + let windows = runtime_is_windows(); + newest_llamacpp_tag_with_asset(&body, rocm_version, windows).with_context(|| { + format!( + "none of the {LLAMACPP_RELEASES_SEARCHED} newest {LLAMACPP_ROCM_STABLE_REPO} \ + releases ships {}", + llamacpp_rocm_stable_asset_name("", rocm_version, windows) + ) + }) } /// Read `resources/backend_versions.json`'s pinned `llamacpp.rocm-stable` build tag. @@ -1018,8 +1109,8 @@ mod tests { #[test] fn plain_semver_accepts_release_versions_only() { - assert!(looks_like_plain_semver("10.0.0")); - assert!(looks_like_plain_semver("7.13.0")); + assert!(looks_like_plain_semver("7.14.0")); + assert!(looks_like_plain_semver("10.1.0")); assert!(!looks_like_plain_semver("7.14.0a20260601")); assert!(!looks_like_plain_semver("10.0")); assert!(!looks_like_plain_semver("10.0.0.1")); @@ -1033,8 +1124,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752" }, - "therock": { "version": "7.13.0", "architectures": ["gfx1151"] }, + "llamacpp": { "rocm-stable": "b11580" }, + "therock": { "version": "10.1.0", "architectures": ["gfx1151"] }, })) .unwrap(), ) @@ -1042,18 +1133,18 @@ mod tests { assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); - write_backend_versions_therock_version(&path, "10.0.0").unwrap(); + write_backend_versions_therock_version(&path, "7.14.0").unwrap(); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); // Every other key, including sibling keys under `therock`, is preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["therock"]["architectures"][0], "gfx1151"); fs::remove_dir_all(&dir).ok(); @@ -1070,7 +1161,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_therock_version(&path), None); - assert!(write_backend_versions_therock_version(&path, "10.0.0").is_err()); + assert!(write_backend_versions_therock_version(&path, "7.14.0").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1082,8 +1173,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752", "vulkan": "b9747" }, - "therock": { "version": "7.13.0" }, + "llamacpp": { "rocm-stable": "b11580", "vulkan": "b11393" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -1091,20 +1182,20 @@ mod tests { assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()) + Some("b11580".to_owned()) ); - write_backend_versions_llamacpp_tag(&path, "b10952").unwrap(); + write_backend_versions_llamacpp_tag(&path, "b12003").unwrap(); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()) + Some("b12003".to_owned()) ); // Sibling keys, including the unrelated `llamacpp.vulkan` tag and the // whole `therock` object, are preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["vulkan"], "b9747"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["vulkan"], "b11393"); + assert_eq!(value["therock"]["version"], "10.1.0"); fs::remove_dir_all(&dir).ok(); } @@ -1120,7 +1211,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_llamacpp_tag(&path), None); - assert!(write_backend_versions_llamacpp_tag(&path, "b10952").is_err()); + assert!(write_backend_versions_llamacpp_tag(&path, "b12003").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1292,7 +1383,7 @@ mod tests { #[test] fn align_honors_the_disabled_flag_without_touching_the_pin_or_the_injected_steps() { // This pins `align`'s own half of the opt-out contract in isolation: the pin - // and llama.cpp tag stay byte-identical and `align`/`latest_tag` are never + // and llama.cpp tag stay byte-identical and `align`/`newest_tag_with_asset` are never // called. The real production gate -- whether this function is even reached // with `disabled == true` in the first place -- lives one level up, in // `prepare_llamacpp_backend_for_active_rocm_impl`; see @@ -1300,14 +1391,14 @@ mod tests { // below for that half. let dir = scratch_dir("align-disabled"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", true, |_manifest, _target, _force_reinstall, _label| { panic!("disabled means no alignment attempt of any kind") @@ -1319,18 +1410,18 @@ mod tests { ); Ok(()) }, - || panic!("disabled means tier 2's tag lookup must not run either"), + |_target| panic!("disabled means tier 2's tag lookup must not run either"), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "the packaged pin must survive untouched when alignment is disabled" ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the packaged llama.cpp tag must survive untouched when alignment is disabled" ); fs::remove_dir_all(&dir).ok(); @@ -1416,8 +1507,8 @@ mod tests { &mut manifest, false, false, - || Ok(Some("10.0.0".to_owned())), - |_path| Some("7.13.0".to_owned()), + || Ok(Some("7.14.0".to_owned())), + |_path| Some("10.1.0".to_owned()), |_manifest| Err(anyhow!("lemond failed to start")), |_manifest, force_reinstall| { assert!( @@ -1899,14 +1990,14 @@ mod tests { fn align_tier1_success_keeps_target_version_pinned() { let dir = scratch_dir("align-tier1-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // A stale backend installed against the OLD pin must not be able to @@ -1916,30 +2007,232 @@ mod tests { true }, |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), - || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + |_target| panic!("tier 1 succeeded; tier 2's tag lookup must not run"), ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); fs::remove_dir_all(&dir).ok(); } + /// One `GET /repos/lemonade-sdk/llama.cpp/releases` entry, trimmed to the + /// fields Tier 2 reads. + fn llamacpp_release(tag: &str, published_at: &str, assets: &[&str]) -> Value { + json!({ + "tag_name": tag, + "draft": false, + "prerelease": false, + "published_at": published_at, + "assets": assets + .iter() + .map(|name| json!({ "name": name, "state": "uploaded" })) + .collect::>(), + }) + } + + /// Synthetic, modelled on the real listing in early November 2026, where + /// b12062 carries only qualification reports and b11997 no ROCm 7.14 + /// archive. The real b12003 ships both platforms; here it ships Linux + /// archives only, so Linux and Windows pick different builds. + fn llamacpp_releases_fixture() -> Value { + json!([ + llamacpp_release( + "b12062", + "2026-11-03T03:41:03Z", + &[ + "qualification-llama-b12062-bin-ubuntu-rocm-10.1-x64.json", + "qualification-llama-b12062-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b12003", + "2026-10-30T03:47:50Z", + &[ + "llama-b12003-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b12003-bin-ubuntu-rocm-7.14-x64.tar.gz", + ], + ), + llamacpp_release( + "b11997", + "2026-10-29T16:11:35Z", + &[ + "llama-b11997-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11997-bin-win-rocm-10.1-x64.zip", + "qualification-llama-b11997-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b11972", + "2026-10-29T00:43:52Z", + &[ + "llama-b11972-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11972-bin-ubuntu-rocm-7.14-x64.tar.gz", + "llama-b11972-bin-win-rocm-10.1-x64.zip", + "llama-b11972-bin-win-rocm-7.14-x64.zip", + ], + ), + ]) + } + + #[test] + fn llamacpp_asset_name_matches_what_lemonade_downloads() { + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "10.1.0", false), + "llama-b11580-bin-ubuntu-rocm-10.1-x64.tar.gz" + ); + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "7.14.0", true), + "llama-b11580-bin-win-rocm-7.14-x64.zip" + ); + } + + #[test] + fn tier2_skips_a_newer_release_that_does_not_ship_the_archive() { + let releases = llamacpp_releases_fixture(); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b12003"), + "the newest release ships only qualification reports" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", true).as_deref(), + Some("b11997"), + "b12003 ships no Windows archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", false).as_deref(), + Some("b12003") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", true).as_deref(), + Some("b11972"), + "b12003 ships no Windows archive and b11997 no ROCm 7.14 archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.13.0", false), + None, + "no release ships ROCm 7.13" + ); + } + + #[test] + fn tier2_picks_by_publication_date_and_skips_unfinished_releases() { + let mut draft = llamacpp_release( + "b12100", + "2026-11-04T00:00:00Z", + &["llama-b12100-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + draft["draft"] = json!(true); + let mut prerelease = llamacpp_release( + "b12101", + "2026-11-04T01:00:00Z", + &["llama-b12101-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + prerelease["prerelease"] = json!(true); + let mut uploading = llamacpp_release( + "b12102", + "2026-11-04T02:00:00Z", + &["llama-b12102-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + uploading["assets"][0]["state"] = json!("open"); + // An asset whose upload state is not reported is not known to be complete. + let mut stateless = llamacpp_release( + "b12103", + "2026-11-04T03:00:00Z", + &["llama-b12103-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + stateless["assets"][0] + .as_object_mut() + .expect("asset object") + .remove("state"); + // Listed out of date order: the listing's order is not trusted. + let releases = json!([ + llamacpp_release( + "b11900", + "2026-10-26T14:08:13Z", + &["llama-b11900-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + draft, + prerelease, + uploading, + stateless, + llamacpp_release( + "b11945", + "2026-10-28T00:47:33Z", + &["llama-b11945-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + ]); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b11945") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&json!({}), "10.1.0", false), + None + ); + } + + #[test] + fn tier2_lookup_asks_for_this_hosts_archive() { + // The newer build ships only the other platform's archive, so only a + // lookup that names this host's platform settles on the older one. + let (this_host, other_host) = if cfg!(windows) { + ( + "llama-b11945-bin-win-rocm-10.1-x64.zip", + "llama-b11960-bin-ubuntu-rocm-10.1-x64.tar.gz", + ) + } else { + ( + "llama-b11945-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11960-bin-win-rocm-10.1-x64.zip", + ) + }; + let releases = json!([ + llamacpp_release("b11960", "2026-10-28T12:41:07Z", &[other_host]), + llamacpp_release("b11945", "2026-10-28T00:47:33Z", &[this_host]), + ]); + let mut requested = String::new(); + let tag = newest_llamacpp_rocm_stable_tag_via("10.1.0", |url| { + requested = url.to_owned(); + Ok(releases) + }) + .unwrap(); + assert_eq!(tag, "b11945"); + assert_eq!( + requested, + "https://api.github.com/repos/lemonade-sdk/llama.cpp/releases?per_page=30" + ); + + let missing = newest_llamacpp_rocm_stable_tag_via("7.13.0", |_| { + Ok(json!([llamacpp_release( + "b11945", + "2026-10-28T00:47:33Z", + &[this_host] + )])) + }) + .unwrap_err(); + assert!(format!("{missing:#}").contains("rocm-7.13"), "{missing:#}"); + let failed = + newest_llamacpp_rocm_stable_tag_via("10.1.0", |_| Err(anyhow!("offline"))).unwrap_err(); + assert!(format!("{failed:#}").contains("offline"), "{failed:#}"); + } + #[test] fn align_falls_through_to_tier2_and_succeeds() { let dir = scratch_dir("align-tier2-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // Both tiers force a reinstall now, so the mock can no longer use @@ -1953,18 +2246,24 @@ mod tests { align_calls > 1 }, |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), - || Ok("b10952".to_owned()), + |target| { + assert_eq!( + target, "7.14.0", + "tier 2 looks for a build that ships the active ROCm version's archive" + ); + Ok("b12003".to_owned()) + }, ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!(align_calls, 2, "both tiers were attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()), + Some("b12003".to_owned()), "tier 2 pins the newer llama.cpp tag" ); fs::remove_dir_all(&dir).ok(); @@ -1978,31 +2277,31 @@ mod tests { // value rather than staying on Tier 2's newer pin. let dir = scratch_dir("align-revert-restores-tag"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the llama.cpp tag must be restored to its original value, not left on Tier 2's pin" ); fs::remove_dir_all(&dir).ok(); @@ -2020,7 +2319,7 @@ mod tests { &path, serde_json::to_vec_pretty(&json!({ "llamacpp": {}, - "therock": { "version": "7.13.0" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -2030,15 +2329,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); @@ -2051,18 +2350,18 @@ mod tests { } #[test] - fn align_skips_tier2_reinstall_when_pinned_tag_is_already_latest() { + fn align_skips_tier2_reinstall_when_the_pinned_tag_is_the_one_found() { let dir = scratch_dir("align-tier2-tag-unchanged"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { align_calls += 1; @@ -2075,15 +2374,16 @@ mod tests { ); Ok(()) }, - // Already the latest tag: Tier 2's own reinstall must not be attempted. - || Ok("b9752".to_owned()), + // The pinned tag is the build Tier 2 found, which Tier 1 already tried: + // Tier 2's own reinstall must not be attempted. + |_target| Ok("b11580".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!(align_calls, 1, "only tier 1 was attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "reverted to the original pinned version" ); fs::remove_dir_all(&dir).ok(); @@ -2096,7 +2396,7 @@ mod tests { // path, and every sibling warning in this function is best-effort. let dir = scratch_dir("align-revert-write-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut fallback_called = false; @@ -2105,8 +2405,8 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { fs::write( @@ -2121,7 +2421,7 @@ mod tests { assert!(force_reinstall); Ok(()) }, - || bail!("network unavailable"), + |_target| bail!("network unavailable"), ); assert_eq!( @@ -2142,7 +2442,7 @@ mod tests { // the pre-existing backend must come back rather than staying deleted. let dir = scratch_dir("align-revert-fallback-install-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let llamacpp = dir.join("bin").join("llamacpp"); let server = platform_binary_name("llama-server"); @@ -2153,15 +2453,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); bail!("network unavailable") }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert!( diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index ed975f763e..bb5c9348d1 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -142,9 +142,10 @@ pub(crate) fn serve_direct_after_router_failure( // would record the service as failed for good (`refresh_from_engine_state` // ignores later states once a record is `failed`), so the state says // `starting` for the whole fallback and goes to `failed` if the fallback - // does not produce a running server. The router's server pid goes too: - // `identity_from_state` prefers it over the launcher's, so leaving the dead - // router's pid in place would point a `stop` during the fallback at a + // does not produce a running server (unless the file has become unusable, + // which refuses the mark and is left as it was). The router's server pid + // goes too: `identity_from_state` prefers it over the launcher's, so leaving + // the dead router's pid in place would point a `stop` during the fallback at a // process that is gone (or, once reused, at an unrelated one) instead of the // launcher that is about to start the direct server. crate::state::merge_json_state( @@ -408,7 +409,7 @@ fn serve_direct_llama_server_within( let mut command = direct_llama_server_command(request, server, model_path); // Packaged llama-server is raw llama.cpp — it does not read `LEMONADE_API_KEY`. // When `rocm serve` protects a public endpoint, hand llama-server the *existing* - // CLI-managed 0600 key file via `--api-key-file` (a path, not the value, so the + // CLI-managed owner-only key file via `--api-key-file` (a path, not the value, so the // secret never lands in the process table). Reusing the managed file rather than // writing a copy keeps the key's lifecycle owned by `rocm serve` — created before // spawn, deleted on stop — with no stale plaintext copy left behind on teardown. @@ -522,9 +523,10 @@ impl DirectServerSteps for RealDirectServerSteps<'_> { /// Own a just-spawned direct llama-server. Every step after the spawn, state /// writes and the final wait included, runs under one guard: any failure stops and -/// reaps the child (`Child` does not kill on drop) and marks the state `failed`, -/// so the server can never keep running with no state that stop and health can -/// reach. +/// reaps the child (`Child` does not kill on drop), so the server can never keep +/// running with no state that stop and health can reach, and then marks the state +/// `failed`. A state file that is empty, torn or not a JSON object refuses that +/// mark and is left as it was. fn supervise_direct_server( child: &mut Child, state_path: &Path, @@ -1970,11 +1972,11 @@ mod tests { #[test] fn direct_serve_binary_finds_build_numbered_nesting() { // Some Lemonade llama.cpp releases extract one level deeper than - // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. + // `/llama-server`, e.g. `rocm-stable/llama-b11580/llama-server`. let dir = scratch_dir("find-server-nested"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let nested = llamacpp.join("rocm-stable").join("llama-b9752"); + let nested = llamacpp.join("rocm-stable").join("llama-b11580"); fs::create_dir_all(&nested).unwrap(); let server = nested.join(platform_binary_name("llama-server")); fs::write(&server, b"x").unwrap(); diff --git a/engines/lemonade/src/footprint.rs b/engines/lemonade/src/footprint.rs index 9acdf26ccb..e7608e274f 100644 --- a/engines/lemonade/src/footprint.rs +++ b/engines/lemonade/src/footprint.rs @@ -135,6 +135,40 @@ pub(crate) fn plan_install_preflight( }) } +/// The runtime folder of a tree of the pinned Lemonade already extracted where +/// this install goes, or `None` when the install would download and extract one. +/// +/// Linux only outside tests, like its one caller, the host-library refusal +/// ([`crate::host_libraries`]): Windows has no such check. +#[cfg(any(target_os = "linux", test))] +pub(crate) fn pinned_tree_in_place(paths: &AppPaths, env_root: Option<&Path>) -> Option { + let runtime_dir = runtime_dir_in(&lemonade_root(paths, env_root)); + let extraction = needs_extraction( + false, + installed_runtime_version(&runtime_dir).as_deref(), + rocm_deps::LEMONADE_VERSION, + lemond_path_in(&runtime_dir).is_file(), + ); + (!extraction).then_some(runtime_dir) +} + +/// What a host-library refusal of an install left undone, for the branch the +/// install was on: a fresh install had a download ahead of it, while a tree of +/// the pinned version already in place had nothing to fetch and is left as it +/// is, though it cannot start either. +#[cfg(any(target_os = "linux", test))] +pub(crate) fn host_refusal_outcome(installed_tree: Option<&Path>) -> String { + match installed_tree { + None => "Nothing was downloaded.".to_owned(), + Some(tree) => format!( + "Nothing was changed: the Lemonade {} already installed in {} stays as it is, \ + but its server cannot start on this host either.", + rocm_deps::LEMONADE_VERSION, + tree.display() + ), + } +} + /// Total size of the files under `dir`, best effort: symlinks are not followed /// and anything unreadable counts as zero, because this only sizes the space an /// install is about to free. @@ -491,6 +525,48 @@ mod tests { const GIB: u64 = 1024 * 1024 * 1024; + /// The host-library refusal's last line follows the install's branch: no + /// tree, or a tree of another version, would have been downloaded, so + /// "Nothing was downloaded."; a tree of the pinned version would not, so + /// the line says it was left in place instead. + #[test] + fn the_host_refusal_outcome_follows_whether_the_pinned_tree_is_in_place() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + + let fresh = pinned_tree_in_place(&paths, None); + assert_eq!(fresh, None, "an empty folder has no tree in place"); + assert_eq!( + host_refusal_outcome(fresh.as_deref()), + "Nothing was downloaded." + ); + + plant_current_tree(&base); + let runtime_dir = runtime_dir_in(&base); + crate::install::record_runtime_version(&runtime_dir, "0.0.1").unwrap(); + assert_eq!( + pinned_tree_in_place(&paths, None), + None, + "a tree of another version is replaced, so it counts as not in place" + ); + + crate::install::record_runtime_version(&runtime_dir, rocm_deps::LEMONADE_VERSION).unwrap(); + let in_place = pinned_tree_in_place(&paths, None); + assert_eq!(in_place.as_deref(), Some(runtime_dir.as_path())); + let outcome = host_refusal_outcome(in_place.as_deref()); + assert_eq!( + outcome, + format!( + "Nothing was changed: the Lemonade {} already installed in {} stays as it \ + is, but its server cannot start on this host either.", + rocm_deps::LEMONADE_VERSION, + runtime_dir.display() + ) + ); + assert!(!outcome.contains("downloaded"), "{outcome}"); + } + /// The release-test sequence: Lemonade installed into the data folder, then /// an SDK runtime activated, so the next install targets the runtime's own /// folder. The notice must name the new folder and the copy it is not reusing. diff --git a/engines/lemonade/src/host_libraries.rs b/engines/lemonade/src/host_libraries.rs new file mode 100644 index 0000000000..ff3ceda0e5 --- /dev/null +++ b/engines/lemonade/src/host_libraries.rs @@ -0,0 +1,660 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether this host's glibc and OpenSSL are new enough for the pinned Linux +//! `lemond`, checked before anything is downloaded and before `lemond` starts. +//! +//! Without this check, a `lemond` built against a newer glibc or OpenSSL than +//! the host has exits at once with a dynamic-loader error ("version +//! `GLIBC_2.43' not found"), and the user sees only that the Lemonade server +//! did not become ready. The versions the pinned build needs are recorded next +//! to its archive digest ([`crate::install::LINUX_LEMOND_NEEDS`]); the host's +//! are read from the libraries the dynamic loader would hand `lemond`, as the +//! newest `GLIBC_*` / `OPENSSL_*` version node each one defines, which is the +//! same thing the loader compares. +//! +//! Each library is found through the `ldconfig` cache, with one of three +//! results ([`HostLibrary`]): +//! +//! - a version, read from the library the cache lists; +//! - missing: the cache was read and lists `libc.so.6`, but not this library's +//! soname. A host with only OpenSSL 1.1 has no `libssl.so.3`, so this is how +//! such a host reads, and it refuses, since `lemond` cannot load at all; +//! - unknown: no cache could be read, the cache lists not even `libc.so.6`, or +//! the library listed did not parse. Unknown never refuses: `lemond`'s own +//! loader error still reaches the startup log tail then, and a false refusal +//! would block a host that runs it fine. + +use std::cmp::Ordering; +use std::fmt; +#[cfg(target_os = "linux")] +use std::sync::OnceLock; + +use anyhow::{Result, bail}; + +/// A dotted library version (`2.38`, `3.0.0`), compared numerically with +/// missing components read as zero, so `3.0` and `3.0.0` are equal. +#[derive(Clone, Copy, Debug)] +pub(crate) struct LibVersion { + parts: [u32; 4], + len: usize, +} + +impl LibVersion { + /// Parse a dotted numeric version of one to four components. + pub(crate) fn parse(text: &str) -> Option { + let mut parts = [0_u32; 4]; + let mut len = 0; + for piece in text.split('.') { + if len == parts.len() || piece.is_empty() || !piece.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + parts[len] = piece.parse().ok()?; + len += 1; + } + Some(Self { parts, len }) + } +} + +impl PartialEq for LibVersion { + fn eq(&self, other: &Self) -> bool { + self.parts == other.parts + } +} + +impl Eq for LibVersion {} + +impl PartialOrd for LibVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for LibVersion { + fn cmp(&self, other: &Self) -> Ordering { + self.parts.cmp(&other.parts) + } +} + +impl fmt::Display for LibVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, part) in self.parts[..self.len].iter().enumerate() { + if index > 0 { + f.write_str(".")?; + } + write!(f, "{part}")?; + } + Ok(()) + } +} + +/// What a pinned Linux `lemond` needs from the host: the newest `GLIBC_*` and +/// `OPENSSL_*` symbol versions it links against, as `objdump -T lemond` lists +/// them, for the archive with digest `archive_sha256`. +pub(crate) struct LemondHostNeeds { + /// The Linux embeddable archive these were read from. A unit test holds it + /// equal to the pinned digest, so a bump cannot keep the old needs. + #[cfg_attr( + not(test), + expect(dead_code, reason = "read only by the test that ties it to the pin") + )] + pub(crate) archive_sha256: &'static str, + pub(crate) glibc: &'static str, + pub(crate) openssl: &'static str, +} + +/// What the probe found for one library. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum HostLibrary { + /// Not read: no loader cache, a cache that lists not even `libc.so.6`, or a + /// library that did not parse. Never refuses. + #[default] + Unknown, + /// The cache was read and lists `libc.so.6`, but not this library's soname. + Missing, + /// The newest version node the library defines. + Version(LibVersion), +} + +/// The host's glibc and OpenSSL, as the probe found them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct HostLibraries { + pub(crate) glibc: HostLibrary, + pub(crate) openssl: HostLibrary, +} + +/// The soname `lemond` loads for glibc and for OpenSSL, and the prefix of the +/// version nodes it defines. +const GLIBC_SONAME: &str = "libc.so.6"; +const OPENSSL_SONAME: &str = "libssl.so.3"; + +/// Why this host cannot run the pinned `lemond`, or `None` when it can or when +/// nothing short of the needs was found. +/// +/// Names each library that falls short with the version needed and what the +/// host has: the newest version node its library defines (which is what the +/// loader compares, and not always the release: Ubuntu 24.04's OpenSSL 3.0.13 +/// defines up to `OPENSSL_3.0.3`), or that the loader cache has no library of +/// that soname. So the message alone says what to upgrade. +pub(crate) fn host_shortfall( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, +) -> Option { + let mut short = Vec::new(); + for (name, soname, prefix, needed, have) in [ + ("glibc", GLIBC_SONAME, "GLIBC_", needs.glibc, host.glibc), + ( + "OpenSSL", + OPENSSL_SONAME, + "OPENSSL_", + needs.openssl, + host.openssl, + ), + ] { + let needed = LibVersion::parse(needed) + .unwrap_or_else(|| panic!("pinned {name} need {needed:?} is not a version")); + match have { + // `have` is the newest version node the library defines, not its + // release (Ubuntu 24.04's OpenSSL 3.0.13 defines up to + // `OPENSSL_3.0.3`), so the message names the node. + HostLibrary::Version(have) if have < needed => short.push(format!( + "{name} {needed} or newer (this host's {soname} provides up to {prefix}{have})" + )), + HostLibrary::Missing => short.push(format!( + "{name} {needed} or newer (this host's loader cache has no {soname})" + )), + HostLibrary::Version(_) | HostLibrary::Unknown => {} + } + } + if short.is_empty() { + return None; + } + Some(format!( + "this host's libraries are too old for the Lemonade {lemonade_version} server: it needs {}", + short.join(" and ") + )) +} + +/// Set (to any value, empty included) to start the pinned `lemond` even where +/// [`host_shortfall`] says the host falls short: the shortfall is then printed +/// as a warning instead of refusing. For a host that provides glibc or OpenSSL 3 +/// outside the loader cache the probe reads (`LD_LIBRARY_PATH`, a Nix or Guix +/// profile, a user prefix, a stale cache). Presence is the signal, as for +/// [`crate::backend_alignment::LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV`]. +pub const LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV: &str = + "ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK"; + +/// Whether the user has turned the host-library check off. +#[cfg(target_os = "linux")] +pub(crate) fn host_library_check_disabled() -> bool { + std::env::var_os(LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV).is_some() +} + +/// What the host-library check decided. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum HostCheck { + /// Nothing short of the needs was found. + Pass, + /// The host falls short, but the check is turned off: go ahead and say so. + Warn(String), + /// The host falls short: stop, with this message. + Refuse(String), +} + +/// Decide from [`host_shortfall`]: refuse with its message, then `outcome` +/// (what the refusal left undone, such as "Nothing was downloaded."), then how +/// to turn the check off; or, when `check_disabled`, warn instead. +pub(crate) fn host_check( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, + outcome: &str, + check_disabled: bool, +) -> HostCheck { + let Some(shortfall) = host_shortfall(lemonade_version, needs, host) else { + return HostCheck::Pass; + }; + if check_disabled { + return HostCheck::Warn(format!( + "warning: {shortfall}; going ahead because \ + {LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV} is set." + )); + } + HostCheck::Refuse(format!( + "{shortfall}. {outcome} If this host provides these libraries outside the \ + loader cache (LD_LIBRARY_PATH, a Nix or Guix profile), set \ + {LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV}=1 to skip this check." + )) +} + +/// Apply [`host_check`]: fail on a refusal, print a warning to stderr. +pub(crate) fn refuse_if_host_too_old( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, + outcome: &str, + check_disabled: bool, +) -> Result<()> { + match host_check(lemonade_version, needs, host, outcome, check_disabled) { + HostCheck::Pass => Ok(()), + HostCheck::Warn(warning) => { + eprintln!("{warning}"); + Ok(()) + } + HostCheck::Refuse(message) => bail!(message), + } +} + +/// The newest numeric version among `names` that start with `prefix` +/// (`GLIBC_2.38` with prefix `GLIBC_`). Names such as `GLIBC_PRIVATE` or the +/// library's own soname entry are skipped. +pub(crate) fn newest_version_node<'a>( + names: impl IntoIterator, + prefix: &str, +) -> Option { + names + .into_iter() + .filter_map(|name| std::str::from_utf8(name).ok()) + .filter_map(|name| name.strip_prefix(prefix)) + .filter_map(LibVersion::parse) + .max() +} + +/// The path `ldconfig -p` gives for `soname` on x86-64, the only Linux +/// architecture the embeddable is installed on. +pub(crate) fn x86_64_library_path<'a>(ldconfig_cache: &'a str, soname: &str) -> Option<&'a str> { + ldconfig_cache.lines().find_map(|line| { + let (left, path) = line.split_once("=>")?; + let left = left.trim(); + let (name, flags) = left.split_once(' ')?; + (name == soname && flags.contains("x86-64")).then(|| path.trim()) + }) +} + +/// Read the host's libraries from `ldconfig -p`'s text (`None` when it could not +/// be run), with `read_version(path, prefix)` reading one library's newest +/// version node. Pure apart from `read_version`, so a test plants the cache. +pub(crate) fn host_libraries_from_cache( + cache: Option<&str>, + read_version: impl Fn(&str, &str) -> Option, +) -> HostLibraries { + let Some(cache) = cache else { + return HostLibraries::default(); + }; + // A cache that lists not even the C library says nothing about the others. + let cache_lists_libc = x86_64_library_path(cache, GLIBC_SONAME).is_some(); + let probe = |soname: &str, prefix: &str| match x86_64_library_path(cache, soname) { + Some(path) => read_version(path, prefix).map_or(HostLibrary::Unknown, HostLibrary::Version), + None if cache_lists_libc => HostLibrary::Missing, + None => HostLibrary::Unknown, + }; + HostLibraries { + glibc: probe(GLIBC_SONAME, "GLIBC_"), + openssl: probe(OPENSSL_SONAME, "OPENSSL_"), + } +} + +/// The version node names an ELF shared library defines (`SHT_GNU_VERDEF`). +#[cfg(target_os = "linux")] +fn defined_version_names(bytes: &[u8]) -> Option> { + use object::read::elf::ElfFile64; + + let file = ElfFile64::::parse(bytes).ok()?; + let endian = file.endian(); + let sections = file.elf_section_table(); + let (mut defs, link) = sections.gnu_verdef(endian, bytes).ok()??; + let strings = sections.strings(endian, bytes, link).ok()?; + let mut names = Vec::new(); + while let Some((_, mut aux)) = defs.next().ok()? { + if let Some(first) = aux.next().ok()? { + names.push(first.name(endian, strings).ok()?); + } + } + Some(names) +} + +/// The newest `prefix` version node the library at `path` defines. +#[cfg(target_os = "linux")] +fn library_version(path: &str, prefix: &str) -> Option { + let bytes = std::fs::read(path).ok()?; + newest_version_node(defined_version_names(&bytes)?, prefix) +} + +/// Debug/test override of what [`probe_host_libraries`] reports, so a +/// black-box scenario reaches the refusal on a host whose libraries are new. +/// Each takes a version, or `missing` for a library the cache does not list. +#[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] +const HOST_GLIBC_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_GLIBC"; +#[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] +const HOST_OPENSSL_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_OPENSSL"; + +/// This host's glibc and OpenSSL, as the libraries the loader cache lists for +/// `libc.so.6` and `libssl.so.3` define them. Probed once per process: the +/// install and every `lemond` start ask, and the answer does not change. +#[cfg(target_os = "linux")] +pub(crate) fn probe_host_libraries() -> HostLibraries { + static PROBED: OnceLock = OnceLock::new(); + *PROBED.get_or_init(|| { + #[cfg(feature = "e2e-test-hooks")] + { + let read = |key: &str| match std::env::var(key).ok().as_deref() { + Some("missing") => HostLibrary::Missing, + Some(value) => { + LibVersion::parse(value).map_or(HostLibrary::Unknown, HostLibrary::Version) + } + None => HostLibrary::Unknown, + }; + let hooked = HostLibraries { + glibc: read(HOST_GLIBC_TEST_ENV), + openssl: read(HOST_OPENSSL_TEST_ENV), + }; + if hooked != HostLibraries::default() { + return hooked; + } + } + host_libraries_from_cache(rocm_core::ldconfig_cache().as_deref(), library_version) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NEEDS: LemondHostNeeds = LemondHostNeeds { + archive_sha256: "", + glibc: "2.38", + openssl: "3.0.0", + }; + + fn version(text: &str) -> Option { + Some(LibVersion::parse(text).unwrap()) + } + + /// One library as a test writes it: a version, `missing`, or `None` for + /// unknown. + fn library(text: Option<&str>) -> HostLibrary { + match text { + None => HostLibrary::Unknown, + Some("missing") => HostLibrary::Missing, + Some(text) => HostLibrary::Version(LibVersion::parse(text).unwrap()), + } + } + + fn host(glibc: Option<&str>, openssl: Option<&str>) -> HostLibraries { + HostLibraries { + glibc: library(glibc), + openssl: library(openssl), + } + } + + #[test] + fn versions_compare_numerically_and_ignore_trailing_zeros() { + assert!(version("2.9") < version("2.38")); + assert!(version("2.39") < version("2.43")); + assert!(version("3.0.3") < version("3.2.0")); + assert_eq!(version("3.0"), version("3.0.0")); + assert_eq!(LibVersion::parse("2.2.5").unwrap().to_string(), "2.2.5"); + for bad in ["", "2.", ".2", "2.x", "PRIVATE", "1.2.3.4.5", "-1"] { + assert!(LibVersion::parse(bad).is_none(), "{bad:?} parsed"); + } + } + + /// The states the check can be in, one row each, as the probe can produce + /// them: every library new enough (Ubuntu 24.04 against the 2026.44.1 + /// needs, and exactly at the need); glibc too old (Ubuntu 22.04); OpenSSL 3 + /// missing with glibc too old (Ubuntu 20.04, OpenSSL 1.1 only); OpenSSL 3 + /// missing alone; and unknown versions, which never refuse. + #[test] + fn shortfall_names_each_library_that_falls_short_and_what_the_host_has() { + let rows: [(HostLibraries, Option<&str>); 7] = [ + (host(Some("2.39"), Some("3.0.3")), None), + (host(Some("2.38"), Some("3.0.0")), None), + ( + host(Some("2.35"), Some("3.0.2")), + Some( + "it needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.35)", + ), + ), + ( + host(Some("2.31"), Some("missing")), + Some( + "it needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.31) and \ + OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", + ), + ), + ( + host(Some("2.39"), Some("missing")), + Some( + "it needs OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", + ), + ), + (host(None, Some("3.0.3")), None), + (host(None, None), None), + ]; + for (host, expected) in rows { + let message = host_shortfall("9.9.9", &NEEDS, &host); + match expected { + None => assert_eq!(message, None, "{host:?}"), + Some(expected) => { + let message = message.unwrap_or_else(|| panic!("no refusal for {host:?}")); + assert!( + message.starts_with( + "this host's libraries are too old for the Lemonade 9.9.9 server: " + ), + "{message}" + ); + assert!(message.ends_with(expected), "{message}"); + } + } + } + } + + /// An OpenSSL 3 older than a pin needs is reported by version: no pin needs + /// that today, but 2026.45.3 needed 3.2.0, and Ubuntu 24.04's OpenSSL 3.0.13 defines + /// only up to `OPENSSL_3.0.3`. + #[test] + fn an_openssl_3_older_than_the_need_is_named_by_version() { + let needs = LemondHostNeeds { + archive_sha256: "", + glibc: "2.43", + openssl: "3.2.0", + }; + let message = host_shortfall("9.9.9", &needs, &host(Some("2.39"), Some("3.0.3"))).unwrap(); + assert!( + message.ends_with( + "it needs glibc 2.43 or newer (this host's libc.so.6 provides up to GLIBC_2.39) and \ + OpenSSL 3.2.0 or newer (this host's libssl.so.3 provides up to OPENSSL_3.0.3)" + ), + "{message}" + ); + } + + #[test] + fn refusal_appends_what_it_left_undone_and_how_to_skip_the_check() { + let HostCheck::Refuse(message) = host_check( + "9.9.9", + &NEEDS, + &host(Some("2.35"), None), + "Nothing was downloaded.", + false, + ) else { + panic!("a host short of glibc was not refused"); + }; + assert!( + message.contains("(this host's libc.so.6 provides up to GLIBC_2.35). Nothing was downloaded. If this host"), + "{message}" + ); + assert!( + message.ends_with( + "set ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1 to skip this check." + ), + "{message}" + ); + assert!( + refuse_if_host_too_old("9.9.9", &NEEDS, &host(Some("2.35"), None), "x", false).is_err() + ); + assert_eq!( + host_check( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.3")), + "x", + false + ), + HostCheck::Pass + ); + refuse_if_host_too_old( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.3")), + "x", + false, + ) + .unwrap(); + } + + /// With the check turned off, a host that falls short is let through, and + /// the warning still names what it falls short of and why it went ahead. + #[test] + fn a_disabled_check_warns_instead_of_refusing() { + let missing_ssl = host(Some("2.31"), Some("missing")); + let HostCheck::Warn(warning) = host_check("9.9.9", &NEEDS, &missing_ssl, "x", true) else { + panic!("a disabled check did not warn"); + }; + assert_eq!( + warning, + "warning: this host's libraries are too old for the Lemonade 9.9.9 server: it \ + needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.31) and OpenSSL 3.0.0 or newer \ + (this host's loader cache has no libssl.so.3); going ahead because \ + ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set." + ); + refuse_if_host_too_old("9.9.9", &NEEDS, &missing_ssl, "x", true).unwrap(); + // A host that meets the needs prints nothing either way. + assert_eq!( + host_check( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.3")), + "x", + true + ), + HostCheck::Pass + ); + } + + const LIBC_LINE: &str = + "\tlibc.so.6 (libc6,x86-64, OS ABI: Linux 3.2.0) => /lib/x86_64-linux-gnu/libc.so.6\n"; + const LIBSSL_LINE: &str = "\tlibssl.so.3 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.3\n"; + const LIBSSL11_LINE: &str = + "\tlibssl.so.1.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.1.1\n"; + + /// A reader standing in for the ELF read: `GLIBC_2.31` and `OPENSSL_3.0.3`, + /// or `None` for a library that does not parse. + fn planted_reader(unparseable: &'static str) -> impl Fn(&str, &str) -> Option { + move |path, _prefix| { + if path.ends_with(unparseable) { + None + } else if path.ends_with("libc.so.6") { + LibVersion::parse("2.31") + } else { + LibVersion::parse("3.0.3") + } + } + } + + /// The probe over a planted `ldconfig -p`, one row per state: an OpenSSL + /// 1.1-only host reads OpenSSL as missing and is refused; a host with both + /// reads versions; and an unreadable cache, a cache that lists no C library, + /// and a listed library that does not parse all read unknown. + #[test] + fn a_read_cache_without_libssl_3_reports_openssl_missing_and_refuses() { + let header = "3 libs found in cache `/etc/ld.so.cache'\n"; + let openssl_1_1_only = format!("{header}{LIBC_LINE}{LIBSSL11_LINE}"); + let probed = host_libraries_from_cache(Some(&openssl_1_1_only), planted_reader("-")); + assert_eq!(probed, host(Some("2.31"), Some("missing"))); + let message = host_shortfall("9.9.9", &NEEDS, &probed).expect("refused"); + assert!( + message + .contains("OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)"), + "{message}" + ); + + let both = format!("{header}{LIBC_LINE}{LIBSSL_LINE}"); + assert_eq!( + host_libraries_from_cache(Some(&both), planted_reader("-")), + host(Some("2.31"), Some("3.0.3")) + ); + assert_eq!( + host_libraries_from_cache(Some(&both), planted_reader("libssl.so.3")), + host(Some("2.31"), None) + ); + assert_eq!( + host_libraries_from_cache(None, planted_reader("-")), + HostLibraries::default() + ); + let no_libc = format!("{header}{LIBSSL11_LINE}"); + assert_eq!( + host_libraries_from_cache(Some(&no_libc), planted_reader("-")), + HostLibraries::default() + ); + } + + #[test] + fn newest_version_node_skips_non_numeric_and_other_prefixes() { + let names: [&[u8]; 6] = [ + b"libc.so.6", + b"GLIBC_2.2.5", + b"GLIBC_2.39", + b"GLIBC_2.4", + b"GLIBC_PRIVATE", + b"OPENSSL_3.2.0", + ]; + assert_eq!(newest_version_node(names, "GLIBC_"), version("2.39")); + assert_eq!(newest_version_node(names, "OPENSSL_"), version("3.2.0")); + assert_eq!(newest_version_node(names, "GLIBCXX_"), None); + } + + #[test] + fn library_path_is_the_x86_64_entry_for_exactly_that_soname() { + let cache = "4 libs found in cache `/etc/ld.so.cache'\n\ + \tlibssl.so.3 (libc6) => /usr/lib/i386-linux-gnu/libssl.so.3\n\ + \tlibssl.so.3 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.3\n\ + \tlibssl.so.30 (libc6,x86-64) => /opt/other/libssl.so.30\n\ + \tlibc.so.6 (libc6,x86-64, OS ABI: Linux 3.2.0) => /lib/x86_64-linux-gnu/libc.so.6\n"; + assert_eq!( + x86_64_library_path(cache, "libssl.so.3"), + Some("/lib/x86_64-linux-gnu/libssl.so.3") + ); + assert_eq!( + x86_64_library_path(cache, "libc.so.6"), + Some("/lib/x86_64-linux-gnu/libc.so.6") + ); + assert_eq!(x86_64_library_path(cache, "libcrypto.so.3"), None); + } + + /// The ELF reader against real libraries: this host's glibc defines version + /// nodes from `GLIBC_2.2.5` up, so the newest is at least 2.17, and an + /// OpenSSL 3 `libssl.so.3` defines `OPENSSL_3.0.0` at least. + #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] + #[test] + fn the_running_hosts_glibc_defines_a_readable_version() { + // A host without `ldconfig` (a minimal container, NixOS) has no cache + // to read; the probe reads that as unknown, which other tests cover. + let Some(cache) = rocm_core::ldconfig_cache() else { + eprintln!("skipped: `ldconfig -p` could not be run on this host"); + return; + }; + let path = x86_64_library_path(&cache, "libc.so.6").expect("libc.so.6 listed"); + let found = library_version(path, "GLIBC_").expect("GLIBC_ version nodes"); + assert!(found >= LibVersion::parse("2.17").unwrap(), "{found}"); + // OpenSSL 3 is optional on a test host; where the cache lists it, its + // own version nodes start at `OPENSSL_3.0.0`. + if let Some(path) = x86_64_library_path(&cache, "libssl.so.3") { + let found = library_version(path, "OPENSSL_").expect("OPENSSL_ version nodes"); + assert!(found >= LibVersion::parse("3.0.0").unwrap(), "{found}"); + } + } +} diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index b6617fc215..02f1a34e07 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -37,9 +37,26 @@ const MAX_EMBEDDABLE_SEARCH_DEPTH: usize = 4; /// release has its own digest, so bumping the pin means recording the new pair /// here as well. const EMBEDDABLE_WINDOWS_SHA256: &str = - "50a133bbc35c4f3f8971eafef2c9fe56c4bbbfb0f1032bf8728324d8d8c8a0e1"; + "7ff6c5d6ab612c33690411a7e53d6d0dd62eb515a073375c35015b0778a5a344"; const EMBEDDABLE_LINUX_SHA256: &str = - "bdfd3c3e5d6eda5101c8a32f36e6dd9236ceb9ab2eb66734c32628e6e86e18ac"; + "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90"; + +/// What the `lemond` in the pinned Linux archive needs from the host, read with +/// `objdump -T lemond` (the newest `GLIBC_*` and `OPENSSL_*` versions listed). +/// A host older than either is refused before download and before `lemond` +/// starts; see [`crate::host_libraries`]. Like the digests, these belong to one +/// release: re-read them on every bump and record that archive's digest here, +/// which a unit test holds equal to [`EMBEDDABLE_LINUX_SHA256`]. Lemonade +/// 2026.45.3's `lemond`, for one, needs glibc 2.43 and OpenSSL 3.2.0, which +/// Ubuntu 24.04 does not have; that is why the pin stays at 2026.44.1 +/// (). +#[cfg(any(target_os = "linux", test))] +pub(crate) const LINUX_LEMOND_NEEDS: crate::host_libraries::LemondHostNeeds = + crate::host_libraries::LemondHostNeeds { + archive_sha256: "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90", + glibc: "2.38", + openssl: "3.0.0", + }; /// Embeddable asset token and archive extension for the host this adapter runs /// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also @@ -791,6 +808,41 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + #[test] + fn pinned_archive_digests_are_two_distinct_sha256_hex_strings() { + // A digest is pasted by hand on every bump and only a live download + // checks it, so a dropped character or a value pasted twice would + // otherwise pass every offline test. + for digest in [EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256] { + assert_eq!(digest.len(), 64, "{digest}"); + assert!( + digest + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)), + "{digest}" + ); + } + assert_ne!(EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256); + } + + /// The host-library needs were read from the archive that is pinned: a bump + /// that records a new Linux digest without re-reading `lemond`'s glibc and + /// OpenSSL needs fails here, instead of refusing hosts by the old needs. + #[test] + fn linux_lemond_needs_belong_to_the_pinned_linux_archive() { + assert_eq!( + LINUX_LEMOND_NEEDS.archive_sha256, EMBEDDABLE_LINUX_SHA256, + "re-read the pinned lemond's needs with `objdump -T lemond` and record \ + them for this archive" + ); + for need in [LINUX_LEMOND_NEEDS.glibc, LINUX_LEMOND_NEEDS.openssl] { + assert!( + crate::host_libraries::LibVersion::parse(need).is_some(), + "{need:?} is not a version" + ); + } + } + #[test] fn a_version_change_forces_re_extraction_over_the_existing_runtime() { let pin = rocm_deps::LEMONADE_VERSION; @@ -816,21 +868,16 @@ mod tests { assert_eq!(installed_runtime_version(runtime_dir), None); - record_runtime_version(runtime_dir, "7.13.0").unwrap(); + record_runtime_version(runtime_dir, "1.2.3").unwrap(); let installed = installed_runtime_version(runtime_dir); - assert_eq!(installed.as_deref(), Some("7.13.0")); + assert_eq!(installed.as_deref(), Some("1.2.3")); assert!(!needs_extraction( false, installed.as_deref(), - "7.13.0", - true - )); - assert!(needs_extraction( - false, - installed.as_deref(), - "7.14.0", + "1.2.3", true )); + assert!(needs_extraction(false, installed.as_deref(), "1.3.0", true)); } #[test] @@ -852,7 +899,7 @@ mod tests { data_dir: env_root.path().join("data"), // no manifest ever written here cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; let root = lemonade_root(&paths, Some(env_root.path())); let runtime_dir = runtime_dir_in(&root); fs::create_dir_all(&runtime_dir).unwrap(); @@ -900,7 +947,7 @@ mod tests { data_dir: env_root.path().join("data"), cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; // A real, well-formed archive (unlike the stub above): this test // exercises the extraction branch, so `extract_archive` must succeed. @@ -1016,13 +1063,13 @@ mod tests { fs::create_dir_all(&backend).unwrap(); fs::write(backend.join("llama-server"), b"installed backend").unwrap(); - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#; let source = dir.join("source-archive"); write_embeddable_with_pins(&source, packaged); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1087,12 +1134,20 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Lemonade version of the restore tests' manifests: synthetic, so a bump + /// of the real pin does not touch these tests. + const TEST_LEMONADE_VERSION: &str = "1.2.3"; + /// Puts `source` where `prepare_embeddable_with` leaves the verified - /// archive of the `11.5.1` test manifests: `/downloads/`. + /// archive of the [`TEST_LEMONADE_VERSION`] test manifests: `/downloads/`. fn cache_archive(root: &Path, source: &Path) { let downloads = root.join("downloads"); fs::create_dir_all(&downloads).unwrap(); - fs::copy(source, downloads.join(embeddable_archive_name("11.5.1"))).unwrap(); + fs::copy( + source, + downloads.join(embeddable_archive_name(TEST_LEMONADE_VERSION)), + ) + .unwrap(); } fn assert_no_pin_temp_files(dir: &Path) { @@ -1109,13 +1164,13 @@ mod tests { } /// Runs the restore against a fake embeddable archive that ships - /// `{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}`, + /// `{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}`, /// with `current` as the pin file already on disk. Returns what the restore /// returned and the pin file's content afterwards. fn restore_over(tag: &str, current: &str) -> (Option, String) { restore_over_packaged( tag, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, current, ) } @@ -1142,7 +1197,7 @@ mod tests { let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1191,8 +1246,8 @@ mod tests { let (backup, after) = restore_over("restore-keeps-user-edits", current); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["therock"]["version"], "10.1.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); assert_eq!(value["custom"]["x"], 1, "{after}"); assert!(backup.is_some()); @@ -1200,7 +1255,7 @@ mod tests { #[test] fn restore_is_a_no_op_when_only_other_entries_differ_from_the_packaged_file() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752","vulkan":"b1111"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580","vulkan":"b1111"}}"#; let (backup, after) = restore_over("restore-noop-other-keys", current); assert!(backup.is_none(), "the alignment pins already match"); @@ -1211,26 +1266,26 @@ mod tests { fn restore_removes_a_pin_the_packaged_file_does_not_have() { // A packaged file without `llamacpp.rocm-stable` must not leave the // kept alignment pin behind; the user's other entries still survive. - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"vulkan":"b9747"}}"#; - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"vulkan":"b11393"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; let (backup, after) = restore_over_packaged("restore-removes-pin", packaged, current); assert!(backup.is_some(), "the kept rocm-stable pin is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); assert!(value["llamacpp"].get("rocm-stable").is_none(), "{after}"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] fn restore_happens_when_only_the_rocm_stable_pin_drifted() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; let (backup, after) = restore_over("restore-only-rocm-stable", current); assert!(backup.is_some(), "a drifted llama.cpp tag alone is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] @@ -1239,7 +1294,7 @@ mod tests { assert!(backup.is_some()); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } /// A runtime tree whose pin file holds a stale kept pin, and the manifest @@ -1257,7 +1312,7 @@ mod tests { .unwrap(); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1307,7 +1362,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); cache_archive(&root, &source); @@ -1338,7 +1393,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); @@ -1362,7 +1417,7 @@ mod tests { ); let value: serde_json::Value = serde_json::from_str(&fs::read_to_string(&pins).unwrap()).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); assert_eq!( value["custom"]["x"], 1, "the concurrent writer's other entries survive" @@ -1384,7 +1439,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 78ea6c164c..112a2c5ffe 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -29,6 +29,9 @@ mod backend_alignment; mod device_policy; mod direct_llama; mod footprint; +// Linux only: Windows `lemond` has no glibc or OpenSSL floor to check. +#[cfg(any(target_os = "linux", test))] +mod host_libraries; mod install; mod llamacpp_backend; mod process; @@ -62,8 +65,12 @@ const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTAL const DEFAULT_LOG_TAIL_LINES: usize = 200; /// How long a stop waits for the server to actually exit after each signal /// before reporting a timeout (or, under `force`, escalating to `SIGKILL`). -const STOP_GRACE: Duration = Duration::from_secs(10); -/// Lemonade state identifies the actual llama-server process directly. +pub(crate) const STOP_GRACE: Duration = Duration::from_secs(10); +/// The scope a stop by recorded state uses. The recorded `server_pid` is the +/// `llama-server` itself on the direct-serve path, but `lemond` on the router +/// path, which runs the model in a `llama-server` child of its own. A +/// `Single` stop of `lemond` that escalates to `SIGKILL` can therefore leave +/// that child running; widening the scope is a follow-up. const STOP_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Single; #[derive(Parser)] @@ -189,8 +196,7 @@ pub fn run_cli() -> Result<()> { gpu_selection: crate::device_policy::parse_gpu_selection_arg(gpu.as_deref())?, })?)?, CommandKind::Stdio => { - let envelope = read_request()?; - print_json(&handle_envelope(envelope))?; + print_json(&answer_stdio(&read_request()?))?; } CommandKind::ServeHttp { service_id, @@ -255,6 +261,11 @@ pub fn builtin_serve_http( } fn handle_envelope(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope { + // Refuse a request made for a protocol this engine cannot serve before its + // payload is read: in another protocol the payload may mean something else. + if let Some(refusal) = rocm_engine_protocol::incompatible_request_response(&envelope) { + return refusal; + } match envelope.method { EngineMethod::Detect => { deserialize_and_respond::(envelope.payload, |_| { @@ -299,7 +310,10 @@ where Ok(response) => EngineResponseEnvelope::success(response), Err(error) => EngineResponseEnvelope::failure("request_failed", format_error(&error)), }, - Err(error) => EngineResponseEnvelope::failure("invalid_payload", error.to_string()), + Err(error) => EngineResponseEnvelope::failure( + rocm_engine_protocol::INVALID_PAYLOAD_CODE, + error.to_string(), + ), } } @@ -412,6 +426,22 @@ fn install_response(request: InstallRequest) -> Result { .env_root .as_deref() .map(normalize_runtime_path_for_host); + // Refused before anything is downloaded: a `lemond` built against a newer + // glibc or OpenSSL than the host has exits at once with a loader error, + // which used to reach the user only as "did not become ready". + // The line saying what the refusal left undone follows the branch: an + // install whose pinned tree is already in place would have downloaded + // nothing anyway. + #[cfg(target_os = "linux")] + crate::host_libraries::refuse_if_host_too_old( + rocm_deps::LEMONADE_VERSION, + &crate::install::LINUX_LEMOND_NEEDS, + &crate::host_libraries::probe_host_libraries(), + &crate::footprint::host_refusal_outcome( + crate::footprint::pinned_tree_in_place(&paths, env_root.as_deref()).as_deref(), + ), + crate::host_libraries::host_library_check_disabled(), + )?; // Announced and space-checked before anything is downloaded: a fresh tree // is a multi-gigabyte install, and `rocm serve` reaches this on its own. if let Some(preflight) = crate::footprint::plan_install_preflight( @@ -623,10 +653,12 @@ fn serve_http(request: ServeHttpRequest) -> Result<()> { } /// Run a serve and make its outcome visible in the persisted state: any error, -/// whichever step raised it, leaves a state file that exists at `failed` rather -/// than at the `starting`/`running` it last recorded, and the error is returned -/// unchanged. Servers spawned inside [`serve_with_context`] are stopped by the -/// code that spawned them, which owns the child. +/// whichever step raised it, marks a state file that exists `failed` rather +/// than leaving the `starting`/`running` it last recorded, and the error is +/// returned unchanged. A state file that is empty, torn or not a JSON object +/// refuses the mark and is left as it was. Servers spawned inside +/// [`serve_with_context`] are stopped by the code that spawned them, which owns +/// the child. fn serve_http_resolving( mut request: ServeHttpRequest, resolve: impl FnOnce(&mut ServeHttpRequest) -> Result, @@ -846,12 +878,21 @@ pub(crate) fn current_unix_millis() -> u128 { .as_millis() } -fn read_request() -> Result { +/// The request text on stdin, read to its end. It is decoded by +/// [`answer_stdio`], which judges its protocol version first. +fn read_request() -> Result { let mut buffer = String::new(); std::io::stdin() .read_to_string(&mut buffer) .context("failed to read stdin for engine request")?; - serde_json::from_str(&buffer).context("failed to parse engine request envelope") + Ok(buffer) +} + +/// The answer to the request `text` read on stdin: its protocol version is +/// judged before its method or payload is decoded (see +/// [`rocm_engine_protocol::answer_engine_request`]), then it is handled. +fn answer_stdio(text: &str) -> EngineResponseEnvelope { + rocm_engine_protocol::answer_engine_request(text, handle_envelope) } fn print_json(value: &T) -> Result<()> { @@ -1604,4 +1645,15 @@ mod tests { assert_eq!(response.canonical_model_id, DEFAULT_MODEL); } } + + /// The engine protocol contract, the same check every engine runs: see + /// [`rocm_engine_protocol::engine_contract`]. + #[test] + fn stdio_answers_hold_the_engine_protocol_contract() { + rocm_engine_protocol::engine_contract::assert_engine_holds_protocol_contract( + "lemonade", + answer_stdio, + handle_envelope, + ); + } } diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 48a9caf91b..3679d5c733 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -166,47 +166,130 @@ pub(crate) fn ensure_best_llamacpp_backend_with_parent( /// Parse `lemonade backends` table output into `(backend, status)` pairs for the /// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. +/// +/// Lemonade also reports two update states, read here as the state they act +/// like: `update_available` (an installed build that still runs, with a newer +/// release out) as `installed`, and `update_required` (an installed build +/// Lemonade will not run until it is reinstalled, e.g. after the pin in +/// `backend_versions.json` moved) as `installable`, so it is reinstalled rather +/// than reused or dropped. Lemonade 2026.44.1's own `Action` for such a row is +/// `lemonade backends install :`, the command rocm-cli runs, +/// and its `BackendManager::install_backend` has no early return for an +/// installed backend while online; for `llamacpp:rocm` it also removes and +/// fetches TheRock again (`src/cpp/server/system_info.cpp` and +/// `src/cpp/server/backend_manager.cpp` at that tag). The other state Lemonade defines, `action_required` +/// (used by other recipes), leaves the row out. +/// +/// A row that starts in the first column and has a state in its Status column +/// names its recipe; an indented row continues the recipe above it with another +/// backend. Any other line, such as a warning, is skipped and leaves the current +/// recipe as it was, even when its text contains a state word: the state is read +/// at the column the header names, not from anywhere in the line. The recipe is read from +/// the layout rather than matched against a list of known recipe names: each +/// Lemonade release adds recipes (`llamacpp-hrx`, `ds4`, `thenoise` in +/// 2026.44), and a row for a recipe missing from such a list used to be read as +/// one more `llamacpp` backend, along with every indented row below it. fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ - "flm", - "kokoro", - "llamacpp", - "ryzenai-llm", - "sd-cpp", - "vllm", - "whispercpp", - "embeddings", - ]; - const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; - let mut current_recipe = ""; + fn is_state(token: &str) -> bool { + token.starts_with("update_available") + || matches!( + token, + "installed" | "installable" | "update_required" | "unsupported" | "action_required" + ) + } + fn usable_status(token: &str) -> Option<&'static str> { + // Lemonade pads the status column to 16 characters with no separator + // after it, so the one 16-character state runs straight into the + // message (`update_availableNewer upstream release ...`). + if token.starts_with("update_available") { + return Some("installed"); + } + match token { + "installed" => Some("installed"), + "installable" | "update_required" => Some("installable"), + "unsupported" => Some("unsupported"), + _ => None, + } + } + // A recipe name as Lemonade spells one: lower case, digits, `-` and `_`. + // Rules out a warning's first word (`Warning:`) in a table with no header. + fn is_recipe_name(token: &str) -> bool { + token + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_') + } + // Each whitespace-separated token with the byte offset it starts at. + fn tokens_at(line: &str) -> Vec<(usize, &str)> { + let mut tokens = Vec::new(); + let mut start = None; + for (index, ch) in line.char_indices() { + match (ch.is_whitespace(), start) { + (true, Some(from)) => { + tokens.push((from, &line[from..index])); + start = None; + } + (false, None) => start = Some(index), + _ => {} + } + } + if let Some(from) = start { + tokens.push((from, &line[from..])); + } + tokens + } + // Where the header puts the Status column, once the header is read. + let mut status_column: Option = None; + let mut current_recipe = String::new(); let mut result = Vec::new(); for line in output.lines() { - let tokens: Vec<&str> = line.split_whitespace().collect(); - let Some(&first) = tokens.first() else { + let tokens = tokens_at(line); + let Some(&(_, first)) = tokens.first() else { continue; }; - if first == "Recipe" || first.starts_with("---") { + if first == "Recipe" { + status_column = line.find("Status"); continue; } - // A row either starts with a recipe name (recipe + its first backend) or, - // for grouped recipes, with a backend name continuing the previous recipe. - let (backend, rest) = if RECIPES.contains(&first) { - current_recipe = match RECIPES.iter().find(|r| **r == first) { - Some(r) => r, - None => current_recipe, - }; - match tokens.get(1) { - Some(backend) => (*backend, &tokens[2..]), - None => continue, - } - } else { - (first, &tokens[1..]) + if first.starts_with("---") { + continue; + } + let continues_recipe = line.starts_with(char::is_whitespace); + let backend_index = usize::from(!continues_recipe); + let Some(&(_, backend)) = tokens.get(backend_index) else { + continue; }; + // The row's state: with a header, the first token that starts at or + // after the Status column (a field wider than its column pushes the + // status right, never left); without one, the first state word after + // the backend. A line whose state slot holds no state is not a row, + // such as a warning Lemonade printed mid-table, even one with a state + // word in its text, and is skipped without ending the recipe above it, + // so the indented rows that follow still count for that recipe. + let state = match status_column { + Some(column) => tokens + .iter() + .skip(backend_index + 1) + .find(|(offset, _)| *offset >= column) + .map(|(_, token)| *token) + .filter(|token| is_state(token)), + None if continues_recipe || is_recipe_name(first) => tokens + .iter() + .skip(backend_index + 1) + .map(|(_, token)| *token) + .find(|token| is_state(token)), + None => None, + }; + let Some(state) = state else { + continue; + }; + if !continues_recipe { + first.clone_into(&mut current_recipe); + } if current_recipe != LLAMACPP_RECIPE { continue; } - if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { - result.push((backend.to_owned(), (*status).to_owned())); + if let Some(status) = usable_status(state) { + result.push((backend.to_owned(), status.to_owned())); } } result @@ -421,6 +504,160 @@ vllm rocm unsupported Requires Linux ); } + /// `lemonade backends` as Lemonade 2026.44.1 prints it on a Linux host with + /// an AMD GPU: a `Tier` and a `Format` column, and recipes (`ds4`, + /// `llamacpp-hrx`, `thenoise`) no earlier release listed. + const LEMONADE_2026_44_BACKENDS_TABLE: &str = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +acestep rocm experimental native installable Backend is supported but not installed. lemonade backends install acestep:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install acestep:vulkan +ds4 rocm guest native installable Backend is supported but not installed. lemonade backends install ds4:rocm +flm npu core native installable Backend is supported but not installed. lemonade backends install flm:npu +kokoro cpu core native installable Backend is supported but not installed. lemonade backends install kokoro:cpu +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install llamacpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install llamacpp:vulkan +llamacpp-hrx hrx experimental native installable Backend is supported but not installed. lemonade backends install llamacpp-hrx:hrx +moonshine cpu core python installable Backend is supported but not installed. lemonade backends install moonshine:cpu +onnxruntime cpu experimental native installable Backend is supported but not installed. lemonade backends install onnxruntime:cpu +openmoss vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install sd-cpp:vulkan +thenoise rocm core python installable Backend is supported but not installed. lemonade backends install thenoise:rocm +thinksound rocm experimental native installable Backend is supported but not installed. lemonade backends install thinksound:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install thinksound:vulkan +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +"; + + #[test] + fn parses_only_llamacpp_rows_from_the_2026_44_table() { + let backends = parse_llamacpp_backend_statuses(LEMONADE_2026_44_BACKENDS_TABLE); + assert_eq!( + backends, + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)) + ); + } + + /// Lines that are not rows between a `llamacpp` row and its indented rows + /// do not end the `llamacpp` rows: a warning with no state word, one whose + /// text contains a state word (`installed`), and one whose first word looks + /// like a recipe name. With a header the state is read at its Status column. + #[test] + fn a_warning_line_inside_the_table_does_not_end_the_llamacpp_rows() { + let table = "\ +Recipe Backend Tier Format Status Message/Version Action +--------------------------------------------------------------------------------------------------------------- +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu +Warning: could not reach GitHub to check for updates +Warning: llamacpp:rocm is installed but outdated +lemonade is installed but update_required for llamacpp:rocm + rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm + vulkan core native installed b11580 +sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm +"; + let expected = vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ]; + assert_eq!(parse_llamacpp_backend_statuses(table), expected); + } + + /// Without a header there is no Status column to read, so a first-column + /// line names a recipe only if its first word is shaped like one: a + /// `Warning:` line with a state word in it is still skipped. + #[test] + fn a_warning_with_a_state_word_does_not_end_the_llamacpp_rows_without_a_header() { + let table = "\ +llamacpp cpu installable Backend is supported but not installed. +Warning: llamacpp:rocm is installed but outdated + rocm update_required Backend update is required before use. + vulkan installed b11580 +sd-cpp cpu installable Backend is supported but not installed. +"; + assert_eq!( + parse_llamacpp_backend_statuses(table), + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + } + + /// A field wider than its column pushes the Status column right; the state + /// is still the first token at or after the header's Status column. + #[test] + fn a_field_wider_than_its_column_still_finds_the_state() { + let table = "\ +Recipe Backend Tier Format Status Message/Version +llamacpp rocm core native-overflowing installed b11580 +"; + assert_eq!( + parse_llamacpp_backend_statuses(table), + vec![("rocm".to_owned(), "installed".to_owned())] + ); + } + + #[test] + fn another_recipes_backend_is_never_read_as_a_llamacpp_backend() { + // Plain `lemonade backends` (no `--all`) leaves unsupported backends + // out, so on a host with no GPU backend for llama.cpp its only row is + // `cpu`. The indented `vulkan` row belongs to `openmoss`; read as a + // `llamacpp` backend, it would select a GPU backend Lemonade does not + // offer for llama.cpp here. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu +openmoss rocm experimental native installable Backend is supported but not installed. lemonade backends install openmoss:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!(backends, vec![("cpu".to_owned(), "installable".to_owned())]); + assert_eq!(select_best_llamacpp_backend(&backends), None); + } + + #[test] + fn update_states_read_as_the_state_they_act_like() { + // `update_required`: the installed build will not run until it is + // reinstalled, so it must be installed again, not reused. An + // `update_available` build still runs and is reused; Lemonade prints + // that 16-character state with no space before the message. An + // `action_required` row is left out, whatever words its message holds. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm + vulkan core native update_availableNewer upstream release available: b11400 lemonade backends install llamacpp:vulkan + system core native action_required Use the installed driver instead. - +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!( + backends, + vec![ + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)), + "an update_required rocm backend is reinstalled, not dropped for vulkan" + ); + } + #[test] fn selects_vulkan_when_rocm_unsupported() { let backends = vec![ diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index fb49ecccb6..e68732967a 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -76,7 +76,9 @@ pub(crate) enum LogMode { } /// `spawn_lemond` with the parent runtime environment supplied by the caller, so -/// tests need not read or create anything under the real user dirs. +/// tests need not read or create anything under the real user dirs. Every +/// spawn goes through here, so the host-library refusal below covers the +/// router serve and the helper `lemond` too. pub(crate) fn spawn_lemond_with_parent( manifest: &LemonadeInstallManifest, host: &str, @@ -85,6 +87,93 @@ pub(crate) fn spawn_lemond_with_parent( process_env: &LemonadeProcessEnvironment, parent: &ParentRuntimeEnvironment, log_mode: LogMode, +) -> Result { + #[cfg(target_os = "linux")] + return spawn_lemond_on( + &crate::host_libraries::probe_host_libraries(), + crate::host_libraries::host_library_check_disabled(), + manifest, + host, + port, + log_path, + process_env, + parent, + log_mode, + ); + #[cfg(not(target_os = "linux"))] + spawn_lemond_unchecked( + manifest, + host, + port, + log_path, + process_env, + parent, + log_mode, + ) +} + +/// [`spawn_lemond_with_parent`] on Linux, with the host's libraries and the +/// opt-out given so a test can drive the refusal on a host whose libraries are +/// new. +#[cfg(target_os = "linux")] +#[allow(clippy::too_many_arguments)] +fn spawn_lemond_on( + host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, + log_mode: LogMode, +) -> Result { + refuse_lemond_on_too_old_host(manifest, host_libraries, check_disabled)?; + spawn_lemond_unchecked( + manifest, + host, + port, + log_path, + process_env, + parent, + log_mode, + ) +} + +/// Refuse to start the pinned Linux `lemond` on a host whose glibc or OpenSSL +/// is older than it needs (see [`crate::host_libraries`]), rather than let it +/// exit with a loader error that reads as a readiness timeout. +/// +/// Applies only to a tree of the pinned version: the recorded needs belong to +/// that archive, and a tree left by an older pin is replaced by the next +/// install, which runs the same check before downloading. +#[cfg(target_os = "linux")] +fn refuse_lemond_on_too_old_host( + manifest: &LemonadeInstallManifest, + host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, +) -> Result<()> { + if manifest.version != rocm_deps::LEMONADE_VERSION { + return Ok(()); + } + crate::host_libraries::refuse_if_host_too_old( + &manifest.version, + &crate::install::LINUX_LEMOND_NEEDS, + host_libraries, + "Lemonade was not started.", + check_disabled, + ) +} + +#[allow(clippy::too_many_arguments)] +fn spawn_lemond_unchecked( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, + log_mode: LogMode, ) -> Result { #[cfg(windows)] if let Some(log_path) = log_path { @@ -825,6 +914,155 @@ mod tests { } } + /// A `lemond` stand-in that records it was started, in a fresh directory. + #[cfg(target_os = "linux")] + fn marker_lemond(version: &str) -> (tempfile::TempDir, LemonadeInstallManifest, PathBuf) { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("started"); + let lemond = dir.path().join("lemond"); + fs::write( + &lemond, + format!("#!/bin/sh\ntouch '{}'\n", marker.display()), + ) + .unwrap(); + fs::set_permissions(&lemond, fs::Permissions::from_mode(0o755)).unwrap(); + let manifest = LemonadeInstallManifest { + version: version.to_owned(), + runtime_dir: dir.path().to_path_buf(), + lemond, + ..test_manifest() + }; + (dir, manifest, marker) + } + + /// Start the stand-in through `spawn_lemond`'s Linux path, with the host's + /// libraries given rather than probed. + /// + /// A script written a moment ago can fail to exec with `ETXTBSY` when + /// another test thread forks while the write handle is still open; that, + /// and only that, is retried. + #[cfg(target_os = "linux")] + fn start_on( + manifest: &LemonadeInstallManifest, + host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, + ) -> Result<()> { + let mut attempts = 0; + let parent_root = tempfile::tempdir().unwrap(); + let mut child = loop { + match spawn_lemond_on( + host_libraries, + check_disabled, + manifest, + "127.0.0.1", + 1, + None, + &LemonadeProcessEnvironment::default(), + &ParentRuntimeEnvironment::rooted_at(parent_root.path()), + LogMode::Fresh, + ) { + Err(error) + if attempts < 50 + && error.chain().any(|cause| { + cause + .downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(libc::ETXTBSY) + }) => + { + attempts += 1; + std::thread::sleep(Duration::from_millis(10)); + } + result => break result?, + } + }; + child.wait()?; + Ok(()) + } + + /// Host libraries as a test writes them: a version, or `missing` for a + /// library the loader cache does not list. + #[cfg(target_os = "linux")] + fn libraries(glibc: &str, openssl: &str) -> crate::host_libraries::HostLibraries { + use crate::host_libraries::{HostLibrary, LibVersion}; + let library = |text: &str| match text { + "missing" => HostLibrary::Missing, + text => HostLibrary::Version(LibVersion::parse(text).unwrap()), + }; + crate::host_libraries::HostLibraries { + glibc: library(glibc), + openssl: library(openssl), + } + } + + /// On a host older than the pinned `lemond` needs (Ubuntu 20.04: glibc 2.31 + /// and OpenSSL 1.1 only), the refusal names what each short library needs + /// and what the host has, says `lemond` was not started, and it really was + /// not: the stand-in never ran. + #[cfg(target_os = "linux")] + #[test] + fn the_pinned_lemond_is_not_started_on_a_host_with_too_old_libraries() { + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + let error = start_on(&manifest, &libraries("2.31", "missing"), false) + .expect_err("a host older than the pinned lemond needs is refused") + .to_string(); + let needs = &crate::install::LINUX_LEMOND_NEEDS; + for expected in [ + format!( + "glibc {} or newer (this host's libc.so.6 provides up to GLIBC_2.31)", + needs.glibc + ), + format!( + "OpenSSL {} or newer (this host's loader cache has no libssl.so.3)", + needs.openssl + ), + format!("the Lemonade {} server", rocm_deps::LEMONADE_VERSION), + ] { + assert!(error.contains(&expected), "missing `{expected}`: {error}"); + } + assert!( + error.contains("Lemonade was not started. If this host provides"), + "{error}" + ); + assert!(!marker.exists(), "lemond ran despite the refusal"); + } + + /// The other rows: a host that meets the needs starts `lemond`, and so does + /// a tree of another version, whose needs this pin does not record. + #[cfg(target_os = "linux")] + #[test] + fn lemond_starts_on_a_new_enough_host_and_for_a_tree_of_another_version() { + let needs = &crate::install::LINUX_LEMOND_NEEDS; + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + start_on(&manifest, &libraries(needs.glibc, needs.openssl), false).unwrap(); + assert!( + marker.exists(), + "lemond did not start on a host meeting its needs" + ); + + let (_dir, manifest, marker) = marker_lemond("0.0.1"); + start_on(&manifest, &libraries("2.31", "missing"), false).unwrap(); + assert!( + marker.exists(), + "a tree of another version was refused by this pin's needs" + ); + } + + /// With `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK` set, the pinned + /// `lemond` starts on a host the check would refuse, as the refusal's own + /// advice promises. + #[cfg(target_os = "linux")] + #[test] + fn a_disabled_check_starts_the_pinned_lemond_on_a_host_it_would_refuse() { + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + start_on(&manifest, &libraries("2.31", "missing"), true).unwrap(); + assert!( + marker.exists(), + "the disabled check still kept lemond from starting" + ); + } + /// Build the real backend-install command for a console state and report /// whether the window was hidden, along with the command itself. fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { diff --git a/engines/lemonade/src/router_serve.rs b/engines/lemonade/src/router_serve.rs index ed0df4fe81..7a81509845 100644 --- a/engines/lemonade/src/router_serve.rs +++ b/engines/lemonade/src/router_serve.rs @@ -9,10 +9,12 @@ //! the way it fails can be tested without a real `lemond`. Production uses //! [`LemondRouterSteps`]. There is deliberately no direct-server step here: any //! failure is returned to the caller as an error and the server is stopped. The -//! engine state is then left at `failed`, except for a model load or +//! engine state is then marked `failed`, except for a model load or //! verification failure: that is tagged [`RouterLoadFailure`] and does not write //! `failed`, so the caller can decide, on Linux, whether the selected GPU -//! backend can serve the model directly instead and own the final state. +//! backend can serve the model directly instead and own the final state. A +//! state file that is empty, torn or not a JSON object refuses the `failed` +//! mark like any other merge and is left as it was. use anyhow::{Context, Result, anyhow}; use serde_json::json; @@ -72,8 +74,11 @@ pub(crate) fn serve_router( ) -> Result<()> { let server_pid = steps.spawn_server()?; // Every step after the spawn is inside this one fallible block, so a failure - // anywhere in it (state writes included) stops the server and marks the state - // `failed`; the server has no `Drop` and would otherwise keep running. + // anywhere in it (state writes included) stops the server, which has no `Drop` + // and would otherwise keep running. The state is then marked `failed`, except + // after a load failure, which leaves it to the fallback (below). When the + // state file is itself unusable (empty, torn or not a JSON object), that mark + // is refused like any merge and the file is left as it was. let outcome = (|| -> Result<()> { crate::state::write_running_state_for_model( request, @@ -265,8 +270,10 @@ impl RouterSteps for LemondRouterSteps<'_> { if let Some(mut child) = self.child.take() { // The whole tree: a router that failed to load a model can leave a // backend process holding the device memory, and the fallback starts - // a second server on the same device and port straight after. - let _ = rocm_core::terminate_process_tree(child.id()); + // a second server on the same device and port straight after. The + // stop escalates to SIGKILL after the grace period, so the wait below + // returns even for a server that ignores SIGTERM. + let _ = crate::state::terminate_pid(child.id(), true); let _ = child.wait(); } } @@ -303,12 +310,16 @@ mod tests { SmokeUnreachable, /// The state file cannot be written once the server is spawned. StateWrite, - /// The state file becomes unwritable between verification and the ready merge. + /// The state file is torn ([`TORN_STATE`]) between verification and the + /// ready merge, as a writer that rewrites it in place could leave it. ReadyMerge, WaitServer, FailedExit, } + /// A state document cut short mid-write: what [`Fail::ReadyMerge`] leaves. + const TORN_STATE: &[u8] = br#"{"status": "running", "server_pid": 4000000, "server_st"#; + struct FakeSteps { runtime: LemonadeRuntime, fail: Fail, @@ -394,8 +405,7 @@ mod tests { // Asked for while the ready state is assembled, i.e. after every // verification step and before the merge is written. if self.fail == Fail::ReadyMerge { - std::fs::remove_file(&self.state_path).unwrap(); - std::fs::create_dir(&self.state_path).unwrap(); + std::fs::write(&self.state_path, TORN_STATE).unwrap(); } None } @@ -580,10 +590,8 @@ mod tests { std::fs::create_dir(&request.state_path).unwrap(); } let mut steps = FakeSteps::new(temporary.path(), fail); - let error = format!( - "{:#}", - serve_router(&request, &mut steps, ROUTER_NAME).expect_err(expected) - ); + let failure = serve_router(&request, &mut steps, ROUTER_NAME).expect_err(expected); + let error = format!("{failure:#}"); assert!(error.contains(expected), "{error}"); // The spawned server is stopped exactly once, and nothing runs after it. assert_eq!( @@ -598,6 +606,24 @@ mod tests { "{:?}", steps.calls ); + if fail == Fail::ReadyMerge { + // Refused as torn, not failed for some other reason, and true to its + // word: neither the ready merge nor the `failed` mark that follows + // replaced the file, so the torn bytes the merge found are still there. + let refused = failure + .downcast_ref::() + .unwrap_or_else(|| panic!("the merge must refuse the torn file: {error}")); + assert_eq!(refused.refusal, rocm_core::JsonStateRefusal::NotJson); + assert!( + request.state_path.is_file(), + "the state file is still a file" + ); + assert_eq!( + std::fs::read(&request.state_path).unwrap(), + TORN_STATE, + "the torn state file must be left exactly as it was" + ); + } if fail == Fail::WaitServer { // The state file is writable here, so the failure is recorded. let state = crate::state::read_service_state(&request.state_path).unwrap(); @@ -612,7 +638,12 @@ mod tests { #[test] fn ready_state_merge_failure_stops_the_server() { - assert_post_spawn_failure_stops_server(Fail::ReadyMerge, "failed to write"); + // The merge reads the file first, under the state directory's lock, and + // refuses a torn one rather than replace it. + assert_post_spawn_failure_stops_server( + Fail::ReadyMerge, + "left it untouched and recorded nothing", + ); } /// The ready write fails after the model loaded and verified, and the server @@ -763,6 +794,95 @@ mod tests { assert!(steps.child.is_none()); } + /// A `lemond` that ignores SIGTERM, with a backend child that ignores it + /// too, is still stopped and reaped: `stop_server` escalates to SIGKILL for + /// the whole tree after the stop grace instead of waiting on the server + /// forever. A watchdog kills both if the stop does not return in time, so a + /// regression fails the test instead of hanging it. + #[cfg(target_os = "linux")] + #[test] + fn real_router_stop_kills_a_server_tree_that_ignores_sigterm() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = request(dir); + let env = LemonadeProcessEnvironment::default(); + let manifest = crate::test_http::fake_runtime::manifest(dir, 0, None); + // The backend child writes its PID once it runs, which is after the + // trap is set in the parent and inherited (`SIG_IGN` survives `exec`). + let backend_pid_file = dir.join("backend.pid"); + crate::test_http::fake_runtime::write_executable( + &manifest.lemond, + &format!( + "#!/bin/sh\ntrap '' TERM\nsh -c 'echo $$ > \"{}\"; exec sleep 300' &\nwait\n", + backend_pid_file.display() + ), + ); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { manifest }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + let server_pid = steps.spawn_server().unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let backend_pid: u32 = loop { + if let Some(pid) = std::fs::read_to_string(&backend_pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + break pid; + } + assert!( + std::time::Instant::now() < deadline, + "the fake lemond never started its backend" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let backend = rocm_core::ProcessIdentity::capture(backend_pid); + + let bound = crate::STOP_GRACE * 2 + Duration::from_secs(10); + let (watchdog_cancel, stopped) = std::sync::mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || { + if stopped.recv_timeout(bound).is_ok() { + return false; + } + for pid in [backend_pid, server_pid] { + let _ = std::process::Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status(); + } + true + }); + steps.stop_server(); + let _ = watchdog_cancel.send(()); + let watchdog_fired = watchdog.join().unwrap(); + let backend_state = rocm_core::identity_state(&backend); + if backend_state == rocm_core::IdentityState::Matches { + let _ = std::process::Command::new("kill") + .args(["-KILL", &backend_pid.to_string()]) + .status(); + } + + assert!( + !watchdog_fired, + "stop_server did not return within {bound:?} for a server ignoring SIGTERM" + ); + assert!(steps.child.is_none()); + assert!( + !Path::new(&format!("/proc/{server_pid}")).exists(), + "the server must be killed and reaped" + ); + assert!( + matches!( + backend_state, + rocm_core::IdentityState::Gone | rocm_core::IdentityState::Recycled + ), + "the backend under the server must be killed too, got {backend_state:?}" + ); + } + /// Backend selection runs `lemonade backends` through the serve request's /// confined parent environment, never the host's: the child must see the /// confined runtime directory, and the answer must come from its listing. diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 56ca91cf06..82c7df0ed3 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -11,6 +11,7 @@ use std::fs; use std::hash::{Hash, Hasher}; use std::net::TcpListener; use std::path::{Path, PathBuf}; +use std::time::Duration; use crate::device_policy::device_policy_name; use crate::install::{LemonadeInstallManifest, LemonadeRuntime}; @@ -136,26 +137,24 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } +/// Replace the service's state file whole. Shares the state directory's lock +/// with every merge (`rocm_core::write_json_state_file`), so a full rewrite +/// never lands inside another writer's merge, and publishes atomically, so a +/// lock-free reader such as `rocm services list` never sees half a document. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(path, serde_json::to_vec_pretty(value)?) - .with_context(|| format!("failed to write {}", path.display())) + rocm_core::write_json_state_file(path, value) } +/// Merge `patch` into the service's state file under the lock the healthcheck's +/// inference latch and `rocmd`'s endpoint publish also take +/// (`rocm_core::merge_json_state_file`), so no merge drops a key another merge +/// wrote — `server_pid` above all, which a later stop needs. A full rewrite +/// ([`write_state`]) is ordered by the same lock but replaces every key. +/// +/// A state file that is empty, torn or not a JSON object is an error and left +/// as it was. pub(crate) fn merge_json_state(path: &Path, patch: &Value) -> Result<()> { - let mut value = read_service_state(path).unwrap_or_else(|_| json!({})); - if !value.is_object() { - value = json!({}); - } - let object = value.as_object_mut().expect("object checked above"); - if let Some(patch) = patch.as_object() { - for (key, value) in patch { - object.insert(key.clone(), value.clone()); - } - } - write_state(path, &value) + rocm_core::merge_json_state_file(path, patch) } pub(crate) fn mark_json_status(path: &Path, status: &str) -> Result<()> { @@ -198,8 +197,58 @@ pub(crate) fn identity_from_state(state: &Value) -> Option bool { - rocm_core::terminate_process(pid).is_ok() +/// Stop a process this engine spawned and still holds as an unreaped child, and +/// report whether it is confirmed gone. +/// +/// On Unix it sends `SIGTERM` to the process tree and waits up to +/// [`STOP_GRACE`](crate::STOP_GRACE) for every process in it to exit. Under +/// `force` it then escalates to `SIGKILL` and waits up to the grace period again, +/// so the call blocks for up to about twice [`STOP_GRACE`](crate::STOP_GRACE) and +/// the caller's following `child.wait()` returns instead of hanging on a process +/// that ignores `SIGTERM`. With `force == false` a process that outlives the +/// grace period is left running. On Windows there is no graceful signal and no +/// tree walk: a forced stop terminates `pid` itself and waits until Windows +/// reports its exit status, and an unforced one sends nothing. `TerminateProcess` +/// completes asynchronously, so a process reported stopped there may still be +/// finishing its termination; the caller's `child.wait()` returns once it has. +/// +/// `false` means the stop is not confirmed, and the reason is written to +/// stderr. That includes a process whose identity could not be confirmed +/// ([`rocm_core::TerminationOutcome::Unverified`]): it is not signalled at all, +/// so a caller that then waits on it blocks until it exits by itself. +/// +/// The process identity is captured here, at call time, not when the process was +/// spawned. That is safe only while the caller holds an unreaped child for `pid` +/// (a `std::process::Child` not yet waited on): until it is reaped, the PID +/// cannot be handed to another process. A Windows `LemondChild::Pid` holds no +/// process handle, so that guarantee does not cover it: if that `lemond` has +/// already exited, its PID may name another process by the time of this call. +/// Do not call this with a PID read back from a state file; stop recorded +/// processes through `rocm_core::terminate_verified` with their recorded +/// identity instead. +pub(crate) fn terminate_pid(pid: u32, force: bool) -> bool { + terminate_pid_within(pid, force, crate::STOP_GRACE) +} + +/// [`terminate_pid`] with the grace period chosen by the caller, so a test can +/// drive the escalation without waiting the production grace period. +fn terminate_pid_within(pid: u32, force: bool, grace: Duration) -> bool { + // `Tree`, unlike the `Single` scope the recorded-state stop uses + // (`crate::STOP_SCOPE`): the children stopped here are often `lemond`, the + // Lemonade router, which runs each loaded model in a `llama-server` child + // process. A forced kill of `lemond` alone would orphan that child, still + // holding the GPU. A child with no children of its own (a direct + // `llama-server`) is stopped exactly as `Single` would stop it. + let outcome = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(pid), + rocm_core::KillScope::Tree, + grace, + force, + ); + if let Some(reason) = outcome.unconfirmed_reason() { + eprintln!("Warning: process {pid} was not confirmed stopped: {reason}."); + } + outcome.stopped() } pub(crate) fn free_local_port() -> Result { @@ -465,4 +514,481 @@ mod tests { Some(("::1".to_owned(), 11435)) ); } + + /// A wrapper merge into a torn state file — the half-written document the + /// old non-atomic writers could leave — fails and keeps the file as it + /// was, rather than replacing it with the patch and losing `server_pid`. + #[test] + fn a_merge_into_a_torn_state_file_fails_and_keeps_it() { + let dir = scratch_dir("torn-state"); + let path = dir.join("svc.json"); + let torn = br#"{"status": "running", "server_pid": 4242, "server_st"#; + fs::write(&path, torn).unwrap(); + + let merge = merge_json_state(&path, &json!({ "status": "ready" })); + let mark = mark_json_status(&path, "stopped"); + + let after = fs::read(&path).unwrap(); + fs::remove_dir_all(&dir).ok(); + for result in [merge, mark] { + let error = format!( + "{:#}", + result.expect_err("a torn state file must be refused") + ); + assert!(error.contains("is not valid JSON"), "{error}"); + assert!( + error.contains("left it untouched and recorded nothing"), + "{error}" + ); + } + assert_eq!(after, torn, "the torn file must be left as it was"); + } + + /// The race behind the lost `server_pid`: the wrapper merging into its own + /// state file while a healthcheck latches its inference probe into the same + /// file. Both must keep the server's identity, so a later stop still + /// targets the server rather than the wrapper. + #[test] + fn wrapper_and_healthcheck_merges_keep_the_server_identity() { + // Few rounds: the race itself is carried by rocm-core's cross-process + // merge test; this one shows the wrapper's own writers take part in it. + const ROUNDS: u64 = 20; + let dir = scratch_dir("merge-race"); + let path = dir.join("svc.json"); + write_state( + &path, + &json!({ + "status": "running", + "pid": 100, + "start_ticks": 111_u64, + "server_pid": 4242, + "server_start_ticks": 99_u64, + }), + ) + .unwrap(); + + let (wrapper, healthcheck) = std::thread::scope(|scope| { + let wrapper = scope.spawn(|| -> Result<()> { + for round in 0..ROUNDS { + merge_json_state(&path, &json!({ "backend_state": "ready", "round": round }))?; + mark_json_status(&path, "ready")?; + } + Ok(()) + }); + let healthcheck = scope.spawn(|| -> Result<()> { + for round in 0..ROUNDS { + rocm_core::merge_json_state_file( + &path, + &json!({ rocm_core::INFERENCE_PROBE_ATTEMPTED_STATE_KEY: round }), + )?; + } + Ok(()) + }); + ( + wrapper.join().expect("wrapper thread"), + healthcheck.join().expect("healthcheck thread"), + ) + }); + + let state = read_service_state(&path); + fs::remove_dir_all(&dir).ok(); + wrapper.expect("every wrapper merge succeeds"); + healthcheck.expect("every healthcheck merge succeeds"); + let state = state.expect("the state file is still a whole document"); + let identity = identity_from_state(&state).expect("an identity to stop"); + assert_eq!(identity.pid, 4242, "a stop must target the server: {state}"); + assert_eq!(identity.start_ticks, Some(99), "{state}"); + assert_eq!(state["round"], ROUNDS - 1, "{state}"); + assert_eq!( + state[rocm_core::INFERENCE_PROBE_ATTEMPTED_STATE_KEY], + ROUNDS - 1, + "{state}" + ); + } + + /// Whether `dir` holds the state directory lock file that only the shared + /// locked writers create. + fn has_state_lock_file(dir: &Path) -> bool { + dir.join(rocm_core::JSON_STATE_LOCK_FILE_NAME).is_file() + } + + /// A full rewrite goes through the shared locked writer (whose locking + /// rocm-core's own tests pin): it leaves the directory's lock file behind, + /// which a write of its own would not. + #[test] + fn a_state_rewrite_goes_through_the_shared_locked_writer() { + let dir = scratch_dir("rewrite-lock"); + let path = dir.join("svc.json"); + + let result = write_state(&path, &json!({ "status": "starting" })); + + let locked = has_state_lock_file(&dir); + let state = read_service_state(&path); + fs::remove_dir_all(&dir).ok(); + result.expect("the rewrite succeeds"); + assert!(locked, "the rewrite must take the state directory's lock"); + assert_eq!( + state.expect("the rewrite was published"), + json!({ "status": "starting" }) + ); + } + + #[test] + fn a_state_merge_goes_through_the_shared_locked_writer() { + let dir = scratch_dir("merge-lock"); + let path = dir.join("svc.json"); + fs::write(&path, br#"{"server_pid": 4242}"#).unwrap(); + + let result = mark_json_status(&path, "ready"); + + let locked = has_state_lock_file(&dir); + let state = read_service_state(&path); + fs::remove_dir_all(&dir).ok(); + result.expect("the merge succeeds"); + assert!(locked, "the merge must take the state directory's lock"); + let state = state.expect("the merge was published"); + assert_eq!(state["server_pid"], 4242, "{state}"); + assert_eq!(state["status"], "ready", "{state}"); + } + + // ---- `terminate_pid`: bounded termination of a spawned child. ---- + + /// A grace period short enough to keep these tests fast, and long enough + /// for a cooperative `sh` to act on `SIGTERM`. + #[cfg(target_os = "linux")] + const TEST_GRACE: Duration = Duration::from_millis(500); + + /// Spawn `sh -c script`, which must print one line once it is ready (for a + /// trapping script: after the trap is installed, so `SIGTERM` can never + /// arrive first), and return the child with that line. + #[cfg(target_os = "linux")] + fn spawn_ready(script: &str) -> (std::process::Child, String) { + use std::io::BufRead; + let mut child = std::process::Command::new("sh") + .args(["-c", script]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + // Not inherited: a child left running by a failing test must not hold + // open the stderr pipe a parent test reads to its end. + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn test child"); + let stdout = child.stdout.take().expect("piped stdout"); + let mut line = String::new(); + std::io::BufReader::new(stdout) + .read_line(&mut line) + .expect("read readiness line"); + (child, line.trim().to_owned()) + } + + /// `SIGKILL` `identity` if it is still that process, so a failing test does + /// not leave a stray process behind. + #[cfg(target_os = "linux")] + fn kill_if_still_running(identity: &rocm_core::ProcessIdentity) { + if rocm_core::identity_state(identity) == rocm_core::IdentityState::Matches { + let _ = std::process::Command::new("kill") + .args(["-KILL", &identity.pid.to_string()]) + .status(); + } + } + + /// `SIGKILL`s each process, if it is still that process, when dropped. + #[cfg(target_os = "linux")] + struct KillOnDrop(Vec); + + #[cfg(target_os = "linux")] + impl Drop for KillOnDrop { + fn drop(&mut self) { + self.0.iter().for_each(kill_if_still_running); + } + } + + #[cfg(target_os = "linux")] + const TRAP_TERM: &str = "trap '' TERM; echo ready; while :; do sleep 1; done"; + + #[cfg(target_os = "linux")] + #[test] + fn forced_terminate_pid_kills_a_child_that_ignores_sigterm() { + use std::os::unix::process::ExitStatusExt; + let (mut child, _) = spawn_ready(TRAP_TERM); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + + let stopped = terminate_pid_within(child.id(), true, TEST_GRACE); + + let status = child.try_wait().expect("try_wait"); + if status.is_none() { + kill_if_still_running(&identity); + let _ = child.wait(); + } + assert!(stopped, "a forced stop must report the child stopped"); + let status = status.expect("the child must have exited after a forced stop"); + assert_eq!( + status.signal(), + Some(libc::SIGKILL), + "a child that ignores SIGTERM can only have been stopped by SIGKILL: {status}" + ); + } + + /// The pattern every caller uses, `terminate_pid` then `child.wait()`, must + /// return within about twice the grace period for a child that ignores + /// `SIGTERM`. Before the fix this hung forever. + #[cfg(target_os = "linux")] + #[test] + fn terminate_then_wait_returns_for_a_child_that_ignores_sigterm() { + let (mut child, _) = spawn_ready(TRAP_TERM); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + let (done, finished) = std::sync::mpsc::channel(); + let teardown = std::thread::spawn(move || { + let stopped = terminate_pid_within(child.id(), true, TEST_GRACE); + let status = child.wait(); + let _ = done.send((stopped, status.is_ok())); + }); + + // The margin is wide: the bound only turns a hang into a failure, and a + // loaded host can be slow to run the stop. + let bound = TEST_GRACE * 2 + Duration::from_secs(10); + let result = finished.recv_timeout(bound); + if result.is_err() { + // Unblock the teardown thread's `wait` so the test fails rather than hangs. + kill_if_still_running(&identity); + } + let _ = teardown.join(); + let (stopped, waited) = + result.unwrap_or_else(|_| panic!("terminate + wait did not return within {bound:?}")); + assert!(stopped); + assert!(waited); + } + + #[cfg(target_os = "linux")] + #[test] + fn unforced_terminate_pid_leaves_a_child_that_ignores_sigterm_running() { + let (mut child, _) = spawn_ready(TRAP_TERM); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + + let stopped = terminate_pid_within(child.id(), false, TEST_GRACE); + let still_running = child.try_wait().expect("try_wait").is_none(); + + kill_if_still_running(&identity); + let _ = child.wait(); + assert!( + !stopped, + "an unforced stop must not claim a surviving child stopped" + ); + assert!(still_running, "an unforced stop must not kill the child"); + } + + #[cfg(target_os = "linux")] + #[test] + fn terminate_pid_stops_a_cooperative_child_with_sigterm() { + use std::os::unix::process::ExitStatusExt; + let (mut child, _) = spawn_ready("echo ready; while :; do sleep 1; done"); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + + let stopped = terminate_pid_within(child.id(), true, Duration::from_secs(5)); + + kill_if_still_running(&identity); + let status = child.wait().expect("wait"); + assert!(stopped); + // Exited on SIGTERM, not SIGKILL: the stop did not escalate. Asserted on + // the signal rather than on timing, which a loaded host makes unreliable. + assert_eq!(status.signal(), Some(libc::SIGTERM), "{status}"); + } + + /// The warning `terminate_pid` writes is checked on the real stderr of a + /// process running it, together with the state it reports: a child that + /// outlives an unforced stop is named, and is still running; a child a + /// forced stop killed is not named. The test re-runs itself in a child + /// process (selected by an environment variable), because a warning written + /// to this process's stderr cannot be read back here. + #[cfg(target_os = "linux")] + #[test] + fn terminate_pid_reports_an_unconfirmed_stop_on_stderr_and_not_a_confirmed_one() { + const ARM: &str = "ROCM_TEST_LEMONADE_STOP_WARNING_ARM"; + if std::env::var_os(ARM).is_some() { + let (mut forced, _) = spawn_ready(TRAP_TERM); + let (mut unforced, _) = spawn_ready(TRAP_TERM); + let forced_identity = rocm_core::ProcessIdentity::capture(forced.id()); + let unforced_identity = rocm_core::ProcessIdentity::capture(unforced.id()); + // Kills both if this arm panics before its own cleanup below. + let _cleanup = KillOnDrop(vec![forced_identity, unforced_identity]); + let forced_stopped = terminate_pid_within(forced.id(), true, TEST_GRACE); + let unforced_stopped = terminate_pid_within(unforced.id(), false, TEST_GRACE); + let running = unforced.try_wait().expect("try_wait").is_none(); + // On a line of its own: the harness has already printed the test's + // name without a line break. + println!("\nforced={} stopped={forced_stopped}", forced.id()); + println!( + "unforced={} stopped={unforced_stopped} running={running}", + unforced.id() + ); + kill_if_still_running(&forced_identity); + kill_if_still_running(&unforced_identity); + let _ = unforced.wait(); + let _ = forced.wait(); + return; + } + + let output = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "state::tests::terminate_pid_reports_an_unconfirmed_stop_on_stderr_and_not_a_confirmed_one", + "--exact", + "--nocapture", + ]) + .env(ARM, "1") + .stdin(std::process::Stdio::null()) + .output() + .expect("re-run the test arm in a child process"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + let pid_after = |prefix: &str| -> u32 { + stdout + .lines() + .find_map(|line| line.strip_prefix(prefix)) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|pid| pid.parse().ok()) + .unwrap_or_else(|| panic!("no {prefix} line in: {stdout}")) + }; + let forced = pid_after("forced="); + let unforced = pid_after("unforced="); + + assert!( + stdout.contains(&format!("forced={forced} stopped=true")), + "{stdout}" + ); + assert!( + stdout.contains(&format!("unforced={unforced} stopped=false running=true")), + "{stdout}" + ); + assert!( + stderr.contains(&format!( + "Warning: process {unforced} was not confirmed stopped: it, or a process under it, \ + was still running when the stop ended." + )), + "{stderr}" + ); + assert!( + !stderr.contains(&format!("process {forced} was not confirmed stopped")), + "a confirmed stop must not warn: {stderr}" + ); + } + + /// `lemond` runs models in `llama-server` children; a forced stop must take + /// those down too, not orphan them. + #[cfg(target_os = "linux")] + #[test] + fn forced_terminate_pid_kills_a_grandchild_that_ignores_sigterm() { + // Both shells ignore SIGTERM (`SIG_IGN` survives `exec`); the inner one + // prints its PID once it is running, after the outer trap is installed. + let (mut child, grandchild_pid) = + spawn_ready(r#"trap '' TERM; sh -c "echo \$\$; exec sleep 300" & wait"#); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + let grandchild = + rocm_core::ProcessIdentity::capture(grandchild_pid.parse().expect("grandchild pid")); + assert_eq!( + rocm_core::identity_state(&grandchild), + rocm_core::IdentityState::Matches, + "grandchild should be running before the stop" + ); + + let stopped = terminate_pid_within(child.id(), true, TEST_GRACE); + let grandchild_state = rocm_core::identity_state(&grandchild); + + kill_if_still_running(&grandchild); + kill_if_still_running(&identity); + let _ = child.wait(); + assert!(stopped); + assert!( + matches!( + grandchild_state, + rocm_core::IdentityState::Gone | rocm_core::IdentityState::Recycled + ), + "a forced stop must kill the child's whole tree, got {grandchild_state:?}" + ); + } + + /// `terminate_pid` itself, as every caller runs it: the production grace + /// period passes before the escalation, the child is then killed, and the + /// caller's `wait` returns within about twice that period. Takes about + /// [`STOP_GRACE`](crate::STOP_GRACE). + #[cfg(target_os = "linux")] + #[test] + fn terminate_pid_escalates_after_the_stop_grace_and_the_callers_wait_returns() { + use std::os::unix::process::ExitStatusExt; + let (mut child, _) = spawn_ready(TRAP_TERM); + let identity = rocm_core::ProcessIdentity::capture(child.id()); + let (done, finished) = std::sync::mpsc::channel(); + let teardown = std::thread::spawn(move || { + let started = std::time::Instant::now(); + let stopped = terminate_pid(child.id(), true); + let status = child.wait(); + let _ = done.send((stopped, status.ok(), started.elapsed())); + }); + + let bound = crate::STOP_GRACE * 2 + Duration::from_secs(5); + let result = finished.recv_timeout(bound); + if result.is_err() { + // Unblock the teardown thread's `wait` so the test fails rather than hangs. + kill_if_still_running(&identity); + } + let _ = teardown.join(); + let (stopped, status, elapsed) = result + .unwrap_or_else(|_| panic!("terminate_pid + wait did not return within {bound:?}")); + assert!(stopped, "a forced stop must report the child stopped"); + assert!( + elapsed >= crate::STOP_GRACE, + "the child must get the whole stop grace before SIGKILL: {elapsed:?}" + ); + let status = status.expect("wait"); + assert_eq!(status.signal(), Some(libc::SIGKILL), "{status}"); + } + + /// The pattern every caller uses, `terminate_pid` then `child.wait()`: the stop + /// is confirmed and the wait returns the exit code `TerminateProcess` set. + /// The wait is part of the check, not an immediate `try_wait`: + /// `TerminateProcess` completes asynchronously, and `terminate_pid` confirms + /// the stop once the exit status is set, which can be just before the process + /// object is signalled. + #[cfg(windows)] + #[test] + fn forced_terminate_pid_stops_a_long_running_child_and_wait_returns() { + let mut child = std::process::Command::new(rocm_core::windows_system_tool("ping.exe")) + .args(["-n", "300", "127.0.0.1"]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + let (done, finished) = std::sync::mpsc::channel(); + let teardown = std::thread::spawn(move || { + let stopped = terminate_pid(pid, true); + // What the stop left behind, for the failure message. + let probe = rocm_core::probe_process(pid); + let code = child.wait().map(|status| status.code()); + let _ = done.send((stopped, probe, code)); + }); + + let bound = crate::STOP_GRACE * 2 + Duration::from_secs(5); + let result = finished.recv_timeout(bound); + if result.is_err() { + // Unblock the teardown thread's wait so the test fails rather than hangs. + let _ = rocm_core::terminate_process(pid); + } + let _ = teardown.join(); + let (stopped, probe, code) = result + .unwrap_or_else(|_| panic!("terminate_pid + wait did not return within {bound:?}")); + assert!( + stopped, + "terminate_pid must confirm the stop; the process probed {probe:?} after it" + ); + // `terminate_process` passes exit code 1 to `TerminateProcess`; a ping + // that ran to completion would exit 0 after five minutes. + assert_eq!( + code.expect("wait"), + Some(1), + "the child must have been terminated; the process probed {probe:?} after terminate_pid" + ); + } } diff --git a/engines/lemonade/tests/stdio_binary.rs b/engines/lemonade/tests/stdio_binary.rs new file mode 100644 index 0000000000..5d2b138012 --- /dev/null +++ b/engines/lemonade/tests/stdio_binary.rs @@ -0,0 +1,67 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `rocm-engine-lemonade stdio` binary, end to end: what it reads on stdin goes +//! through the shared request reader, so a request it cannot serve, or input +//! that is not a request, is answered with an error envelope on stdout and a +//! clean exit. + +use std::io::Write as _; +use std::process::{Command, Stdio}; + +/// Run `rocm-engine-lemonade stdio` with `request` on stdin, on scratch app roots, +/// and return its exit status and the error code of the envelope it printed. +fn answer(label: &str, request: &str) -> (Option, String) { + let root = std::env::temp_dir().join(format!( + "rocm-engine-lemonade-stdio-{label}-{}", + std::process::id() + )); + let mut child = Command::new(env!("CARGO_BIN_EXE_rocm-engine-lemonade")) + .arg("stdio") + .env("ROCM_CLI_CONFIG_DIR", root.join("config")) + .env("ROCM_CLI_DATA_DIR", root.join("data")) + .env("ROCM_CLI_CACHE_DIR", root.join("cache")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start the engine binary"); + child + .stdin + .take() + .expect("piped stdin") + .write_all(request.as_bytes()) + .expect("write the request"); + let output = child + .wait_with_output() + .expect("wait for the engine binary"); + let _ = std::fs::remove_dir_all(&root); + let stdout = String::from_utf8_lossy(&output.stdout); + let envelope: serde_json::Value = serde_json::from_str(&stdout).unwrap_or_else(|error| { + panic!( + "{label}: no envelope on stdout ({error}): {stdout:?}; stderr: {}", + String::from_utf8_lossy(&output.stderr) + ) + }); + let code = envelope["error"]["code"] + .as_str() + .unwrap_or_default() + .to_owned(); + (output.status.code(), code) +} + +#[test] +fn the_stdio_binary_answers_requests_it_cannot_serve() { + assert_eq!( + answer( + "incompatible", + r#"{"protocol_version":"9.0.0","method":"frobnicate","payload":{}}"# + ), + (Some(0), "incompatible_protocol".to_owned()) + ); + assert_eq!( + answer("garbage", "this is not a request"), + (Some(0), "invalid_request".to_owned()) + ); +} diff --git a/engines/vllm/Cargo.toml b/engines/vllm/Cargo.toml index 05102d0f1e..e7c94e7eeb 100644 --- a/engines/vllm/Cargo.toml +++ b/engines/vllm/Cargo.toml @@ -25,3 +25,7 @@ rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } serde.workspace = true serde_json.workspace = true tempfile = "3" + +[dev-dependencies] +# The shared engine protocol contract check (`engine_contract`). +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index 99d7a92462..f1629323b5 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -9,17 +9,20 @@ /// `runtime.rs` and `wheel_builds.rs` can both match against it without either /// importing the other's discover-build code. pub(crate) fn rocm_sdk_series_matches(recorded: &str, table_key: &str) -> bool { - fn series(version: &str) -> Option<(u64, u64)> { - let mut parts = version.trim().split('.'); - let major = parts.next()?.parse().ok()?; - let minor: String = parts - .next()? - .chars() - .take_while(char::is_ascii_digit) - .collect(); - Some((major, minor.parse().ok()?)) - } - series(recorded).is_some_and(|version| Some(version) == series(table_key)) + rocm_series(recorded).is_some_and(|version| Some(version) == rocm_series(table_key)) +} + +/// The `(major, minor)` of a ROCm version string, ignoring patch and any +/// dev/pre-release suffix. +pub(crate) fn rocm_series(version: &str) -> Option<(u64, u64)> { + let mut parts = version.trim().split('.'); + let major = parts.next()?.parse().ok()?; + let minor: String = parts + .next()? + .chars() + .take_while(char::is_ascii_digit) + .collect(); + Some((major, minor.parse().ok()?)) } /// How a [`VllmRocmDiscoverBuild`] row constrains the release discovery may @@ -47,10 +50,26 @@ impl DiscoverVersion { } } +/// Where a [`VllmRocmDiscoverBuild`] row gets its torch, torchvision and +/// torchaudio from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TorchStack { + /// Resolved from the row's `torch_index_url` by series, newest first. + Discover, + /// The versions the SDK install already put in the runtime, checked + /// against the row's series and ROCm line. For an index that serves + /// several ROCm lines at once, where the newest build of a series belongs + /// to the newest line: the nightly index lists about two dozen + /// `+rocm10.3.0aDATE` torch builds after the last `+rocm10.2.0a20261022` + /// one, and PEP 440 has no local-version wildcard to ask for the line. + FromInstalledSdk, +} + /// A ROCm SDK version whose vLLM/flash-attn/amd-aiter wheels aren't published /// under a fixed filename (AMD rotates the dev-tag suffix constantly), so the /// exact wheel must be discovered from the index at install time instead of /// pinned in `wheel_builds::VLLM_ROCM_BUILD_TABLE`. +#[derive(Clone, Copy)] pub(crate) struct VllmRocmDiscoverBuild { pub(crate) rocm_sdk_version: &'static str, /// PEP 425 interpreter tag this row's wheels are published for. AMD builds @@ -87,6 +106,7 @@ pub(crate) struct VllmRocmDiscoverBuild { /// [`verified_torchaudio_release`](crate::wheel_builds::verified_torchaudio_release): set one only for a build confirmed to /// load against the row's torch on real hardware, never as a guess. pub(crate) torchaudio_version: DiscoverVersion, + pub(crate) torch_stack: TorchStack, } /// Every ROCm SDK `major.minor` line the engine installs by live wheel @@ -120,6 +140,7 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // establish, and this row has no coverage on the self-hosted lanes. // `rocm install sdk` reads this field too: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack: TorchStack::Discover, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.1.0", @@ -155,6 +176,39 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // confirmed-working builds instead. `rocm install sdk` pins this same // build for the 10.1 line: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Exact("2.11.0.2"), + torch_stack: TorchStack::Discover, + }, + VllmRocmDiscoverBuild { + rocm_sdk_version: "10.2.0", + python_tag: "cp314", + // Same staging host as 10.1: it publishes the `.rocm102` vLLM builds + // (`0.32.1.dev0+g.d.rocm102`) next to the 10.1 ones. + vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/", + // The torch stack is the SDK's own, fetched again from the index its + // runtime manifest records (see `TorchStack::FromInstalledSdk`); the + // nightly index below is only the fallback for a manifest that + // recorded none. Not the staging + // host's `+rocm10.2.0rcN` torch: that build requires + // `amd-torch-device-gfx*==2.14.0+rocm10.2.0rcN` and + // `rocm==10.2.0rc2`, which staging answers 403 for, so a GPU payload + // for it cannot be installed, and it is a different release than the + // `10.2.0aDATE` SDK that `rocm install sdk` installs. The 10.2 vLLM + // wheel imports (`import vllm._C`) against the SDK's own torch + // `2.14.0+rocm10.2.0a20261022`. + torch_index_url: "https://nightly.repo.amd.com/rocm/whl-next/", + vllm_version_prefix: "0.32", + flash_attn_version_prefix: "2.8", + amd_aiter_version_prefix: "0.1", + // 2.14 or newer, not 2.13: with `torch 2.13.0+rocm10.1.0`, + // `import vllm._C` fails on the undefined symbol + // `_ZN3c104cuda19CUDAErrorLogCaptureC1Ev`; 2.14 imports cleanly. + torch_version_prefix: "2.14", + torchvision_version_prefix: "0.29", + // A series match, not Exact: an Exact pin would make `rocm install + // sdk` treat it as verified for a 10.2 SDK, which nobody has run on a + // GPU. + torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack: TorchStack::FromInstalledSdk, }, ]; @@ -198,7 +252,7 @@ mod tests { .collect(); assert_eq!( versions, - ["10.0.0", "10.1.0"], + ["10.0.0", "10.1.0", "10.2.0"], "the discover-build table's versions or order changed: install routing, the \ launch environment and the fail-closed static fallback all follow it" ); @@ -277,8 +331,28 @@ mod tests { DiscoverVersion::Exact("2.11.0.2") ); + let ten_two = vllm_rocm_discover_build("10.2.0a20261022").expect("10.2 has a discover row"); + assert_eq!( + ten_two.vllm_index_url, + "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/" + ); + // Still read by the torch install and the realign for the 10.2 row, as the + // fallback when the runtime manifest records no index of its own. + assert_eq!( + ten_two.torch_index_url, + "https://nightly.repo.amd.com/rocm/whl-next/" + ); + assert_eq!(ten_two.vllm_version_prefix, "0.32"); + // 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol. + assert_eq!(ten_two.torch_version_prefix, "2.14"); + assert_eq!(ten_two.torchvision_version_prefix, "0.29"); + // The nightly index serves 10.2 and 10.3 torch side by side, so the + // 10.2 row reuses the SDK's own stack; the other rows discover theirs. + assert_eq!(ten_two.torch_stack, TorchStack::FromInstalledSdk); + assert_eq!(ten_one.torch_stack, TorchStack::Discover); + // An unpublished line must not borrow another line's wheels. - assert!(vllm_rocm_discover_build("10.2.0").is_none()); + assert!(vllm_rocm_discover_build("10.3.0").is_none()); assert!(vllm_rocm_discover_build("7.2.3").is_none()); } } diff --git a/engines/vllm/src/endpoint_auth.rs b/engines/vllm/src/endpoint_auth.rs new file mode 100644 index 0000000000..6fd0a76350 --- /dev/null +++ b/engines/vllm/src/endpoint_auth.rs @@ -0,0 +1,433 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Endpoint API-key enforcement for the vLLM server. +//! +//! Setting `VLLM_API_KEY` alone does not protect a vLLM endpoint: vLLM's own +//! check guards only a fixed set of path prefixes (`/v1` and a few others), +//! while the server also registers inference-capable routes outside them — +//! `/invocations`, `/tokenize`, `/detokenize` and more. So whenever an endpoint +//! key is in force, the server is additionally started with +//! `--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`: a small ASGI +//! middleware, shipped inside this binary and written next to the engine's +//! data at launch, that requires the key on every route except +//! `GET`/`HEAD /health` and CORS preflights. +//! +//! vLLM imports the middleware by module path, so the directory it is written +//! to is prepended to the server's `PYTHONPATH`. A failed import aborts vLLM's +//! startup, so a launch that asked for a key fails closed rather than coming up +//! with only the partial check. + +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; + +/// Python module name the middleware is imported as. +const MODULE_NAME: &str = "rocm_cli_vllm_auth"; + +/// Source of the middleware module, compiled into the binary so the copy vLLM +/// imports always matches the adapter that launched it. +const MODULE_SOURCE: &str = include_str!("rocm_cli_vllm_auth.py"); + +/// The `--middleware` import path vLLM is given. +pub(crate) const MIDDLEWARE_IMPORT_PATH: &str = "rocm_cli_vllm_auth.RequireEndpointApiKey"; + +/// Environment variable carrying the key to both vLLM's own check and the +/// middleware. The middleware reads this exact name (`KEY_ENV` in the module). +pub(crate) const KEY_ENV: &str = "VLLM_API_KEY"; + +/// Extra `vllm serve` arguments that install the middleware. +pub(crate) fn middleware_args() -> [String; 2] { + ["--middleware".to_owned(), MIDDLEWARE_IMPORT_PATH.to_owned()] +} + +/// Directory the middleware module is written to under the engine's data +/// directory. Owned by the invoking user like the rest of that tree, so only +/// they can replace the code vLLM imports. +pub(crate) fn module_dir(paths: &rocm_core::AppPaths) -> PathBuf { + paths.engine_dir(crate::ENGINE_NAME).join("python") +} + +/// Write the middleware module into `dir`, replacing any earlier copy. +/// +/// Written to a temporary file and renamed into place, so a concurrent launch +/// importing the module never reads a half-written file. +pub(crate) fn write_module(dir: &Path) -> Result { + fs::create_dir_all(dir).with_context(|| format!("failed to create {}", dir.display()))?; + let path = dir.join(format!("{MODULE_NAME}.py")); + let mut staged = tempfile::NamedTempFile::new_in(dir) + .with_context(|| format!("failed to stage a file in {}", dir.display()))?; + staged + .write_all(MODULE_SOURCE.as_bytes()) + .with_context(|| format!("failed to write {}", path.display()))?; + staged + .persist(&path) + .with_context(|| format!("failed to write {}", path.display()))?; + Ok(path) +} + +/// Require `api_key` on every route of the vLLM server `command` will start. +/// +/// Sets the key, writes the middleware module into `module_dir` and prepends +/// that directory to `PYTHONPATH`. Call after any other `PYTHONPATH` change +/// (see `apply_therock_env`): an entry `command` already sets is extended +/// rather than replaced. The `--middleware` argument itself comes from +/// [`middleware_args`] via `vllm_serve_args`, so the argv stays a pure +/// function of the launch request. +pub(crate) fn protect_endpoint( + command: &mut ProcessCommand, + api_key: &str, + module_dir: &Path, +) -> Result<()> { + write_module(module_dir)?; + command.env(KEY_ENV, api_key); + let pythonpath = middleware_pythonpath( + module_dir, + command_env(command, "PYTHONPATH"), + std::env::var_os("PYTHONPATH"), + )?; + command.env("PYTHONPATH", pythonpath); + Ok(()) +} + +/// How a [`ProcessCommand`] treats one environment variable. +#[derive(Debug, PartialEq, Eq)] +enum EnvOverride { + /// The command leaves it alone; the child inherits this process's value. + Inherited, + /// The command removes it (`env_remove`). + Removed, + /// The command sets it to this value. + Set(OsString), +} + +/// The server's `PYTHONPATH`: `module_dir` first, then whatever the server +/// would otherwise have seen — the value the command sets, nothing if it +/// removes the variable, or `inherited` when it leaves it alone. Dropping the +/// inherited value would silently change which packages the server imports. +fn middleware_pythonpath( + module_dir: &Path, + command_value: EnvOverride, + inherited: Option, +) -> Result { + let current = match command_value { + EnvOverride::Inherited => inherited, + EnvOverride::Removed => None, + EnvOverride::Set(value) => Some(value), + }; + crate::process::prepend_path_entries(&[module_dir.to_path_buf()], current) +} + +/// How `command` treats the environment variable `key`. +fn command_env(command: &ProcessCommand, key: &str) -> EnvOverride { + command + .get_envs() + .find_map(|(name, value)| (name == key).then_some(value)) + .map_or(EnvOverride::Inherited, |value| match value { + Some(value) => EnvOverride::Set(value.to_owned()), + None => EnvOverride::Removed, + }) +} + +/// The value `command` sets `key` to, if it sets one. Shared by this module's +/// tests and the launch-command tests in `process.rs`. +#[cfg(test)] +pub(crate) fn env_of(command: &ProcessCommand, key: &str) -> Option { + match command_env(command, key) { + EnvOverride::Set(value) => Some(value), + EnvOverride::Inherited | EnvOverride::Removed => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Stdio; + + fn entries(value: &OsString) -> Vec { + std::env::split_paths(value).collect() + } + + #[test] + fn middleware_pythonpath_keeps_the_inherited_value_when_the_command_sets_none() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = middleware_pythonpath( + &module_dir, + EnvOverride::Inherited, + Some("/user/site".into()), + )?; + assert_eq!( + entries(&composed), + vec![module_dir, PathBuf::from("/user/site")], + "the server must still see the PYTHONPATH it would have inherited" + ); + Ok(()) + } + + #[test] + fn middleware_pythonpath_prefers_the_command_value_over_the_inherited_one() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = middleware_pythonpath( + &module_dir, + EnvOverride::Set("/sdk/amd_smi".into()), + Some("/user/site".into()), + )?; + assert_eq!( + entries(&composed), + vec![module_dir, PathBuf::from("/sdk/amd_smi")] + ); + Ok(()) + } + + #[test] + fn middleware_pythonpath_honours_a_command_that_removes_the_variable() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = + middleware_pythonpath(&module_dir, EnvOverride::Removed, Some("/user/site".into()))?; + assert_eq!(entries(&composed), vec![module_dir]); + Ok(()) + } + + #[test] + fn command_env_distinguishes_inherited_removed_and_set() { + let mut command = ProcessCommand::new("vllm"); + assert_eq!(command_env(&command, "PYTHONPATH"), EnvOverride::Inherited); + command.env_remove("PYTHONPATH"); + assert_eq!(command_env(&command, "PYTHONPATH"), EnvOverride::Removed); + command.env("PYTHONPATH", "/x"); + assert_eq!( + command_env(&command, "PYTHONPATH"), + EnvOverride::Set("/x".into()) + ); + } + + #[test] + fn protect_endpoint_sets_key_and_makes_the_middleware_importable() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + let mut command = ProcessCommand::new("vllm"); + command.env("PYTHONPATH", "/opt/existing"); + + protect_endpoint(&mut command, "secret-key", &module_dir)?; + + assert_eq!(env_of(&command, KEY_ENV), Some("secret-key".into())); + let pythonpath = env_of(&command, "PYTHONPATH").expect("PYTHONPATH set"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + assert_eq!( + entries, + vec![module_dir.clone(), PathBuf::from("/opt/existing")], + "the middleware directory goes first and an earlier entry is kept" + ); + let written = fs::read_to_string(module_dir.join(format!("{MODULE_NAME}.py")))?; + assert_eq!(written, MODULE_SOURCE); + Ok(()) + } + + #[test] + fn write_module_replaces_a_stale_copy() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join(format!("{MODULE_NAME}.py")); + fs::write(&path, "class RequireEndpointApiKey: pass\n")?; + assert_eq!(write_module(temp.path())?, path); + assert_eq!(fs::read_to_string(&path)?, MODULE_SOURCE); + Ok(()) + } + + #[test] + fn import_path_names_the_shipped_module_and_class() { + let (module, class) = MIDDLEWARE_IMPORT_PATH + .rsplit_once('.') + .expect("module.Class"); + assert_eq!(module, MODULE_NAME); + assert!(MODULE_SOURCE.contains(&format!("\nclass {class}:"))); + assert!(MODULE_SOURCE.contains(&format!("KEY_ENV = \"{KEY_ENV}\""))); + } + + /// Drives the shipped middleware with a stdlib-only ASGI harness, the same + /// way vLLM's `--middleware` loader mounts it (`app.add_middleware(cls)` + /// wraps the app in `cls(app)`). Proves the routes vLLM leaves outside its + /// own key check — `/invocations`, `/tokenize`, ... — answer 401 without + /// the key, and that only `GET`/`HEAD /health` and CORS preflights stay + /// open. + /// + /// Needs `python3` on PATH and fails loudly without it rather than passing + /// vacuously. Linux-only: vLLM serving is Linux/WSL-only in rocm-cli, so + /// the module never runs anywhere else. + #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "vLLM serving (and so this middleware) is Linux/WSL-only" + )] + fn middleware_requires_the_key_on_every_route_but_health() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + write_module(&module_dir)?; + let harness = temp.path().join("harness.py"); + fs::write(&harness, ASGI_HARNESS)?; + + let output = ProcessCommand::new("python3") + .arg(&harness) + .env("PYTHONPATH", &module_dir) + .env(KEY_ENV, "secret-key") + .stdin(Stdio::null()) + .output() + .context("python3 is required to exercise the vLLM auth middleware")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "middleware harness failed\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert!(stdout.contains("ALL OK"), "harness stdout:\n{stdout}"); + Ok(()) + } + + /// Construction must fail when the key is empty, so a protected launch + /// whose key went missing cannot answer a request without checking one. + #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "vLLM serving (and so this middleware) is Linux/WSL-only" + )] + fn middleware_refuses_to_start_without_a_key() -> Result<()> { + let temp = tempfile::tempdir()?; + write_module(temp.path())?; + let output = ProcessCommand::new("python3") + .args([ + "-c", + "import rocm_cli_vllm_auth as m; m.RequireEndpointApiKey(lambda *a: None)", + ]) + .env("PYTHONPATH", temp.path()) + .env_remove(KEY_ENV) + .stdin(Stdio::null()) + .output() + .context("python3 is required to exercise the vLLM auth middleware")?; + assert!( + !output.status.success(), + "constructing without a key must fail" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("VLLM_API_KEY is empty"), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) + } + + const ASGI_HARNESS: &str = r#" +import asyncio +import rocm_cli_vllm_auth as m + +async def inner(scope, receive, send): + if scope["type"] == "websocket": + await send({"type": "websocket.accept"}) + return + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"inner"}) + +async def call(app, method, path, headers=(), kind="http", root_path=""): + sent = [] + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + async def send(message): + sent.append(message) + scope = {"type": kind, "method": method, "path": path, "root_path": root_path, + "headers": [(k.lower().encode(), v.encode()) for k, v in headers]} + await app(scope, receive, send) + first = sent[0] + if first["type"] == "http.response.start": + return first["status"] + return first["type"] + +GOOD = [("Authorization", "Bearer secret-key")] +CASES = [ + # Routes outside vLLM's own guarded prefixes must need the key too. + ("POST", "/invocations", (), 401), + ("POST", "/tokenize", (), 401), + ("POST", "/detokenize", (), 401), + ("POST", "/generative_scoring", (), 401), + ("POST", "/scale_elastic_ep", (), 401), + ("GET", "/metrics", (), 401), + ("GET", "/ping", (), 401), + ("POST", "/v1/chat/completions", (), 401), + ("GET", "/some/future/route", (), 401), + # Wrong, malformed or non-bearer credentials. + ("POST", "/invocations", [("Authorization", "Bearer wrong")], 401), + ("POST", "/invocations", [("Authorization", "Bearer secret-key-and-more")], 401), + ("POST", "/invocations", [("Authorization", "Basic secret-key")], 401), + ("POST", "/invocations", [("Authorization", "secret-key")], 401), + ("POST", "/invocations", [("X-Api-Key", "secret-key")], 401), + # Only GET/HEAD /health is open, and only at exactly that path. + ("GET", "/health", (), 200), + ("HEAD", "/health", (), 200), + ("POST", "/health", (), 401), + ("GET", "/health/", (), 401), + ("GET", "/healthz", (), 401), + # A CORS preflight (OPTIONS carrying both Origin and + # Access-Control-Request-Method) is the only other keyless request. Each + # part alone is not one, and neither are those headers on another method. + ("OPTIONS", "/invocations", (), 401), + ("OPTIONS", "/invocations", + [("Origin", "http://x"), ("Access-Control-Request-Method", "POST")], 200), + ("OPTIONS", "/invocations", [("Origin", "http://x")], 401), + ("OPTIONS", "/invocations", [("Access-Control-Request-Method", "POST")], 401), + ("POST", "/invocations", + [("Origin", "http://x"), ("Access-Control-Request-Method", "POST")], 401), + # The right key opens every route, with any bearer-scheme casing. + ("POST", "/invocations", GOOD, 200), + ("POST", "/tokenize", GOOD, 200), + ("GET", "/metrics", GOOD, 200), + ("POST", "/v1/chat/completions", [("Authorization", "bearer secret-key")], 200), +] + +async def main(): + app = m.RequireEndpointApiKey(inner) + failures = [] + for method, path, headers, want in CASES: + got = await call(app, method, path, headers) + if got != want: + failures.append(f"{method} {path} {headers!r}: want {want}, got {got}") + # root_path is stripped before matching, as vLLM's own check does. + got = await call(app, "GET", "/prefix/health", root_path="/prefix") + if got != 200: + failures.append(f"root_path /health: want 200, got {got}") + # Only a path that starts with root_path has it stripped. A path of the same + # length as root_path but outside it is not the health check, and neither is + # a path that holds root_path anywhere but at its start. + for path in ("/xxxxxx/health", "/health/prefix"): + got = await call(app, "GET", path, root_path="/prefix") + if got != 401: + failures.append(f"{path} outside root_path: want 401, got {got}") + # WebSocket routes are closed before the handshake without the key. + got = await call(app, None, "/v1/realtime", kind="websocket") + if got != "websocket.close": + failures.append(f"websocket without key: want close, got {got}") + got = await call(app, None, "/v1/realtime", GOOD, kind="websocket") + if got != "websocket.accept": + failures.append(f"websocket with key: want accept, got {got}") + # The /health exemption is for HTTP GET/HEAD only, not a WebSocket there. + # This catches an exemption keyed on the path alone (for any scope type). + # Dropping only the scope-type guard in `_exempt` would still pass here, + # because a WebSocket scope carries no method for the GET/HEAD check. + got = await call(app, None, "/health", kind="websocket") + if got != "websocket.close": + failures.append(f"websocket /health without key: want close, got {got}") + # Lifespan events pass straight through. + seen = [] + async def lifespan_inner(scope, receive, send): + seen.append(scope["type"]) + await m.RequireEndpointApiKey(lifespan_inner)({"type": "lifespan"}, None, None) + if seen != ["lifespan"]: + failures.append(f"lifespan not passed through: {seen}") + if failures: + print("\n".join(failures)) + raise SystemExit(1) + print("ALL OK") + +asyncio.run(main()) +"#; +} diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 26453cb1ac..377a5f681e 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -4,20 +4,23 @@ use anyhow::{Context, Result, anyhow, bail}; use rocm_core::{ - AppPaths, command_failure_detail, ensure_uv_binary, split_local_version, uv_command_env, - uv_pip_freeze_args, uv_pip_install_base, + AppPaths, TENSORIZER_PYTHON314_FIX, command_failure_detail, ensure_uv_binary, + split_local_version, tensorizer_override_requirement, uv_command_env, uv_pip_freeze_args, + uv_pip_install_base, }; use rocm_engine_protocol::{InstallRequest, InstallResponse}; use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; use crate::discover_builds::{ - DiscoverVersion, VllmRocmDiscoverBuild, rocm_sdk_series_matches, vllm_rocm_discover_build, + DiscoverVersion, TorchStack, VllmRocmDiscoverBuild, rocm_sdk_series_matches, rocm_series, + vllm_rocm_discover_build, }; use crate::repair::{RepairAssessment, assess_runtime_repair}; use crate::runtime::{ - assessed_python_for_repair, describe_skipped_managed_runtimes, resolve_managed_runtime_python, - resolve_vllm_runtime, runtime_is_managed, vllm_runtime_warnings, + VllmRuntime, assessed_python_for_repair, describe_skipped_managed_runtimes, + recorded_sdk_index_url, resolve_managed_runtime_python, resolve_vllm_runtime, + runtime_is_managed, vllm_runtime_warnings, }; use crate::state::runtime_lock_hash; use crate::wheel_builds::{ @@ -50,6 +53,9 @@ pub(crate) fn install_response(request: InstallRequest) -> Result = Vec::new(); let mut discover_notes: Vec = Vec::new(); + // True whenever this run installed vLLM: a first install, `--reinstall` or a + // repair pass. Only a runtime that was resolved as already installed skips it. + let installed_vllm_now = already_installed.is_none(); let runtime = if let Some(runtime) = already_installed { runtime } else { @@ -89,6 +95,7 @@ pub(crate) fn install_response(request: InstallRequest) -> Result Result Option<&str> { + stderr.lines().rev().find_map(|line| { + let rest = line.trim_start().strip_prefix("File \"")?; + rest.split_once('"').map(|(path, _)| path) + }) +} + +/// Whether `path` lies inside a `tensorizer` package directory, with either +/// path separator. +fn path_is_in_tensorizer_package(path: &str) -> bool { + path.contains("/tensorizer/") || path.contains("\\tensorizer\\") +} + +/// Judges the stderr of a failed `import tensorizer.serialization`. +/// +/// The import pulls in torch, so a torch or shared-library failure has tensorizer +/// frames in its traceback too. A failure is tensorizer's own when the last frame +/// before the exception is inside the tensorizer package and the exception is not +/// a library-load or missing-module error (`ImportError`, `OSError`): that is the +/// shape of the Python 3.14 `KeyError` and of any other tensorizer code that does +/// not run on this interpreter. Everything else is left to the device check. +fn classify_tensorizer_failure(stderr: &str) -> TensorizerProbe { + // Python 3.13+ colourises tracebacks under `FORCE_COLOR` even when piped, which + // wraps the quoted path in `File "..."` in escapes and defeats the frame + // parser. The probe asks for no colour; this is the second line of defence. + let stderr = stderr + .lines() + .map(rocm_core::strip_terminal_control_sequences) + .collect::>() + .join("\n"); + let stderr = stderr.as_str(); + let Some(exception) = stderr.lines().rev().find(|line| !line.trim().is_empty()) else { + return TensorizerProbe::Usable; + }; + let exception = exception.trim(); + let kind = exception.split([':', ' ']).next().unwrap_or_default(); + let is_load_failure = kind.ends_with("ImportError") + || kind.ends_with("OSError") + || kind == "ModuleNotFoundError" + || exception.contains("shared object file") + || exception.contains("DLL load failed"); + let in_tensorizer = last_traceback_frame(stderr).is_some_and(path_is_in_tensorizer_package); + if in_tensorizer && !is_load_failure { + TensorizerProbe::Broken(exception.to_owned()) + } else { + TensorizerProbe::Usable + } +} + +/// The command that imports what vLLM imports from `tensorizer` in `python`. +/// +/// Colour is switched off (a forced-colour environment makes the interpreter wrap +/// the traceback's quoted paths in escape codes), and with `runtime` the command +/// carries the environment the server is launched with (SDK library path, +/// `PYTHONPATH` for a ROCm 10.x discover build, and the rest), so a runtime that +/// needs it to import torch is judged as it will run. +fn tensorizer_probe_command( + python: &Path, + runtime: Option<&VllmRuntime>, +) -> Result { + let mut command = ProcessCommand::new(python); + // Not `-I`: that implies `-E` and would drop the `PYTHONPATH` (and the rest of + // the `PYTHON*` environment) the launch sets for a ROCm 10.x discover build, + // so a module the server finds would read as missing. `-s` keeps the user's + // site directory out, and `PYTHONSAFEPATH` keeps the working directory off + // `sys.path` (Python 3.11+; older interpreters ignore it). + command + .args(["-s", "-c", "import tensorizer.serialization"]) + .env("PYTHONSAFEPATH", "1"); + if let Some(runtime) = runtime { + // A launch environment that cannot be built is a failure of the check, not + // a runtime that imports: probing without it could read as `Usable` a + // failure the server would not have. + crate::process::apply_therock_env(&mut command, runtime) + .context("could not build the environment the server is launched with")?; + } + command + .env_remove("FORCE_COLOR") + .env("NO_COLOR", "1") + .env("PYTHON_COLORS", "0"); + Ok(command) +} + +/// Imports what vLLM imports from `tensorizer` in `python`. +/// +/// vLLM's model loader imports `tensorizer.serialization` at startup and only +/// guards `ImportError`, so any other exception raised inside tensorizer ends +/// the engine core. With `runtime`, the probe runs under the environment the +/// server is launched with (SDK library path and the rest), so a runtime that +/// needs it to import torch is judged as it will run. +fn probe_tensorizer_import( + python: &Path, + runtime: Option<&VllmRuntime>, +) -> Result { + probe_tensorizer_import_within(python, runtime, TENSORIZER_STEP_TIMEOUT) +} + +/// [`probe_tensorizer_import`] with the deadline as a parameter, so a test can +/// exercise the timeout without waiting minutes. +fn probe_tensorizer_import_within( + python: &Path, + runtime: Option<&VllmRuntime>, + limit: std::time::Duration, +) -> Result { + let command = tensorizer_probe_command(python, runtime)?; + let output = rocm_core::process::run_with_timeout(command, limit, "tensorizer import probe") + .with_context(|| format!("failed to import tensorizer with {}", python.display()))?; + if output.status.success() { + return Ok(TensorizerProbe::Usable); + } + Ok(classify_tensorizer_failure(&String::from_utf8_lossy( + &output.stderr, + ))) +} + +/// Installs the fixed tensorizer over vLLM's pin, leaving everything else as it is. +fn upgrade_tensorizer(uv: &Path, paths: &AppPaths, python: &Path) -> Result<()> { + upgrade_tensorizer_within(uv, paths, python, TENSORIZER_STEP_TIMEOUT) +} + +/// [`upgrade_tensorizer`] with the deadline as a parameter, so a test can +/// exercise the timeout without waiting minutes. +fn upgrade_tensorizer_within( + uv: &Path, + paths: &AppPaths, + python: &Path, + limit: std::time::Duration, +) -> Result<()> { + let mut args = uv_pip_install_base(python); + args.push("--no-deps".to_owned()); + args.push(tensorizer_override_requirement()); + let mut command = ProcessCommand::new(uv); + command.args(&args).envs(uv_command_env(paths)); + let output = + rocm_core::process::run_with_timeout(command, limit, "uv pip install for tensorizer") + .context("failed to run uv pip install for tensorizer")?; + if output.status.success() { + return Ok(()); + } + bail!( + "`uv {}` failed: {}", + args.join(" "), + command_failure_detail(&output) + ) +} + +/// The decision behind [`ensure_runtime_tensorizer`], with the probe and the +/// upgrade injected. +/// +/// Only a managed runtime is rocm-cli's to change, so an external one is not +/// probed at all. The probe runs first so an interpreter the pinned release works +/// on keeps exactly what vLLM asked for. +/// +/// Three things can go wrong after a broken tensorizer is found: the upgrade +/// fails, the probe cannot run, or the upgrade leaves tensorizer unimportable. +/// Each is an error when `fatal` (this run installed vLLM, whether a first +/// install, `--reinstall` or a repair, so the runtime would be left unable to +/// serve or unchecked) and otherwise a note carrying the same facts, so a no-op +/// install on a runtime this run did not touch (an offline host, say) stays one. +/// Returns the note describing what was done, if anything. +fn tensorizer_step( + managed: bool, + fatal: bool, + mut probe: impl FnMut() -> Result, + upgrade: impl FnOnce() -> Result<()>, +) -> Result> { + if !managed { + return Ok(None); + } + // A probe that cannot run (spawn failure, timeout) is a failure of the check, + // not a finding. It fails when this run installed vLLM, which must not report + // a runtime it could not look at. On a runtime it did not touch it is a note, + // so a host where the check cannot run keeps the no-op install it had before. + let unchecked = |error: anyhow::Error| -> Result> { + if fatal { + Err(error) + } else { + Ok(Some(format!( + "could not check whether vLLM's tensorizer imports: {error:#}" + ))) + } + }; + let reason = match probe() { + Ok(TensorizerProbe::Broken(reason)) => reason, + Ok(TensorizerProbe::Usable) => return Ok(None), + Err(error) => return unchecked(error), + }; + let requirement = tensorizer_override_requirement(); + if let Err(error) = upgrade() { + let message = format!( + "vLLM's pinned tensorizer cannot be imported ({reason}) and upgrading it to \ + {requirement} failed: {error:#}" + ); + return if fatal { + Err(anyhow::anyhow!("vLLM was installed, but {message}")) + } else { + Ok(Some(format!("{message}; vLLM would fail at startup"))) + }; + } + let after = match probe() { + Ok(after) => after, + Err(error) => return unchecked(error), + }; + match after { + TensorizerProbe::Usable => Ok(Some(format!( + "vLLM pins a tensorizer that cannot be imported under this Python ({reason}); \ + upgraded it to {TENSORIZER_PYTHON314_FIX}, so `uv pip check` reports vLLM's pin as \ + unmet on purpose" + ))), + TensorizerProbe::Broken(after) => { + // An upgrade can fail to help (the exception is not the one 2.12.1 + // fixes). When this run installed vLLM that leaves a runtime that + // cannot serve, so it fails; on a runtime this run did not touch the + // install used to be a no-op and stays one, with the same facts as a note. + let message = format!( + "tensorizer still cannot be imported after upgrading it to {requirement}: \ + {after}. vLLM would fail at startup, so this runtime cannot serve with vLLM." + ); + if fatal { + bail!("vLLM was installed, but {message}") + } + Ok(Some(message)) + } + } +} + +/// Makes sure vLLM's `tensorizer` dependency imports in `runtime`, upgrading it +/// when it does not. +/// +/// vLLM's wheel pins `tensorizer==2.10.1`, which raises +/// `KeyError: 'crypto_stream_salsa20_NONCEBYTES'` at import under Python 3.14 +/// (its constants metaclass reads annotations from the class namespace, where +/// 3.14 no longer puts them; fixed in tensorizer 2.12.1). The ROCm 10.1.0 vLLM +/// build exists for Python 3.14 only, so the pin cannot be avoided by choosing +/// another Python, and an install that reports success would end in an engine +/// core crash at the first `rocm serve`. Runs after an install pass (`installed_now`) +/// and on an existing runtime, so one installed before this check existed is +/// repaired by the next `rocm engines install vllm`. +fn ensure_runtime_tensorizer(runtime: &VllmRuntime, installed_now: bool) -> Result> { + ensure_runtime_tensorizer_with(runtime, installed_now, |python| { + let paths = AppPaths::discover()?; + let uv = + ensure_uv_binary(&paths).context("failed to acquire uv binary for vLLM install")?; + upgrade_tensorizer(&uv, &paths, python) + }) +} + +/// [`ensure_runtime_tensorizer`] with the upgrade injected, so a test can drive +/// the real probe (and its launch environment) against a stand-in interpreter. +fn ensure_runtime_tensorizer_with( + runtime: &VllmRuntime, + installed_now: bool, + upgrade: impl FnOnce(&Path) -> Result<()>, +) -> Result> { + // The managed check lives in `tensorizer_step`, once, so the rule has one home. + let Some(python) = runtime.python_executable.as_deref() else { + return Ok(None); + }; + tensorizer_step( + runtime_is_managed(runtime), + installed_now, + || probe_tensorizer_import(python, Some(runtime)), + || upgrade(python), + ) +} + /// What a vLLM install route resolved: the pins worth surfacing as an install /// warning, and a note for each place the route fell back to a weaker choice /// than the one it prefers. Both are empty for the static-pin route. @@ -279,6 +567,104 @@ fn discover_pinned_requirement_for( ) }) } +/// The row's torch, torchvision and torchaudio pins, in that order. +/// +/// A [`TorchStack::Discover`] row resolves them from its index. A +/// [`TorchStack::FromInstalledSdk`] row takes them from the runtime the SDK +/// install already resolved, because the index it realigns from serves +/// several ROCm lines at once and a series such as `torch==2.14.*` would pick +/// the newest line's build (see [`TorchStack`]). +fn torch_stack_pins( + uv: &Path, + paths: &AppPaths, + python: &Path, + build: &VllmRocmDiscoverBuild, +) -> Result> { + match build.torch_stack { + TorchStack::Discover => ["torch", "torchvision", "torchaudio"] + .into_iter() + .map(|pkg| { + let version = match pkg { + "torch" => DiscoverVersion::Series(build.torch_version_prefix), + "torchvision" => DiscoverVersion::Series(build.torchvision_version_prefix), + _ => build.torchaudio_version, + }; + discover_pinned_requirement(uv, paths, python, build.torch_index_url, pkg, version) + }) + .collect(), + TorchStack::FromInstalledSdk => { + let output = ProcessCommand::new(uv) + .args(uv_pip_freeze_args(python)) + .envs(uv_command_env(paths)) + .output() + .context("failed to launch `uv pip freeze` to read the SDK's torch stack")?; + if !output.status.success() { + bail!( + "`uv pip freeze` failed for {}: {}", + python.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + installed_torch_stack_pins(&String::from_utf8_lossy(&output.stdout), build) + } + } +} + +/// Whether an installed `release` (a `uv pip freeze` version, with any `+local` part) +/// satisfies `wanted`: a series takes any release inside it, an exact pin takes that +/// release and no longer one, with the same meaning as PEP 440 `==release`. +fn stack_release_matches(release: &str, wanted: DiscoverVersion) -> bool { + match wanted { + DiscoverVersion::Series(series) => release.starts_with(&format!("{series}.")), + DiscoverVersion::Exact(exact) => split_local_version(release).0 == exact, + } +} +/// [`torch_stack_pins`] for a [`TorchStack::FromInstalledSdk`] row, from the +/// `uv pip freeze` output of the SDK runtime. +/// +/// Each pin must belong to the row's torch, torchvision and torchaudio series +/// and be built for the row's ROCm line, so a runtime whose torch is not the +/// one the vLLM wheel was built against is refused instead of silently kept. +fn installed_torch_stack_pins(freeze: &str, build: &VllmRocmDiscoverBuild) -> Result> { + let wanted = [ + ("torch", DiscoverVersion::Series(build.torch_version_prefix)), + ( + "torchvision", + DiscoverVersion::Series(build.torchvision_version_prefix), + ), + ("torchaudio", build.torchaudio_version), + ]; + wanted + .into_iter() + .map(|(pkg, wanted_release)| { + let prefix = format!("{pkg}=="); + let pin = freeze + .lines() + .map(str::trim) + .find(|line| line.starts_with(&prefix)) + .with_context(|| { + format!( + "the ROCm SDK runtime has no `{pkg}` installed; vLLM for ROCm {} reuses \ + the SDK's own torch stack, so reinstall the SDK with `rocm install sdk`", + build.rocm_sdk_version + ) + })?; + let release = &pin[prefix.len()..]; + if !stack_release_matches(release, wanted_release) { + bail!( + "the ROCm SDK runtime has `{pin}`, but vLLM for ROCm {} needs `{pkg}` {}", + build.rocm_sdk_version, + match wanted_release { + DiscoverVersion::Series(series) => format!("{series}.x"), + DiscoverVersion::Exact(release) => release.to_owned(), + } + ); + } + ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; + Ok(pin.to_owned()) + }) + .collect() +} /// Runs the dependency-resolving `uv pip install --dry-run` for the discovered /// vLLM pin and returns uv's report, without installing anything. /// @@ -396,6 +782,32 @@ fn dry_run_resolved_pin(stdout: &str, pkg: &str) -> Option { (!version.is_empty()).then(|| format!("{pkg}=={version}")) }) } +/// `err` with the fallbacks this install used appended, so a failure on a fallback +/// names it. The cause comes first, so plain `{}` display leads with the uv failure and +/// not with a note about how the pins or the index were chosen. +fn with_fallback_notes(err: anyhow::Error, notes: &[String]) -> anyhow::Error { + if notes.is_empty() { + err + } else { + anyhow!( + "{err:#}\n\nFallbacks used during this install: {}", + notes.join(" ") + ) + } +} +/// The index the torch stack is fetched from for `build`. +/// +/// A [`TorchStack::FromInstalledSdk`] row reuses the SDK's own builds, so it asks for +/// them from the index the SDK came from (`recorded`, from the runtime manifest) rather +/// than the row's fixed one: the row serves nightly and release SDKs alike, and each +/// one's torch lives only on its own index. A manifest that recorded none, and every +/// row that discovers its stack, keeps the row's `torch_index_url`. +fn torch_index_url_for(build: &VllmRocmDiscoverBuild, recorded: Option<&str>) -> String { + match (build.torch_stack, recorded) { + (TorchStack::FromInstalledSdk, Some(url)) => url.to_owned(), + _ => build.torch_index_url.to_owned(), + } +} /// Builds the `uv pip install` argv for pinned torch alone, scoped to only /// `torch_index_url`. /// @@ -425,7 +837,7 @@ fn dry_run_resolved_pin(stdout: &str, pkg: &str) -> Option { fn vllm_rocm10_discover_torch_install_args( python: &Path, reinstall: bool, - build: &VllmRocmDiscoverBuild, + torch_index_url: &str, torch_pin: &str, ) -> Vec { let mut args = uv_pip_install_base(python); @@ -438,7 +850,7 @@ fn vllm_rocm10_discover_torch_install_args( args.push("--prerelease".to_owned()); args.push("allow".to_owned()); args.push("--index-url".to_owned()); - args.push(build.torch_index_url.to_owned()); + args.push(torch_index_url.to_owned()); args } /// Builds the `uv pip install` argv that realigns torch, torchvision and @@ -460,7 +872,7 @@ fn vllm_rocm10_discover_torch_install_args( /// exists to prevent. fn vllm_rocm10_discover_realign_install_args( python: &Path, - build: &VllmRocmDiscoverBuild, + torch_index_url: &str, pins: &[String], ) -> Vec { let mut args = uv_pip_install_base(python); @@ -475,7 +887,7 @@ fn vllm_rocm10_discover_realign_install_args( args.push("--prerelease".to_owned()); args.push("allow".to_owned()); args.push("--index-url".to_owned()); - args.push(build.torch_index_url.to_owned()); + args.push(torch_index_url.to_owned()); args } /// Builds the `uv pip install` argv for pinned vllm/flash-attn/amd-aiter, @@ -572,10 +984,10 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul let Some(local) = split_local_version(version).1 else { return Ok(()); }; - let Some(local_rocm) = local.strip_prefix("rocm") else { + let Some(matches) = local_rocm_matches(local, rocm_sdk_version) else { return Ok(()); }; - if rocm_sdk_series_matches(local_rocm, rocm_sdk_version) { + if matches { return Ok(()); } bail!( @@ -583,6 +995,39 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul different ROCm release; the index for this ROCm line is serving another SDK's wheels" ) } +/// Whether the ROCm line a local version names is `rocm_sdk_version`'s, or `None` when +/// it has no `rocm` segment at all. A dotted `rocm10.2.0` is read from its first `rocm` segment to the end +/// of the local version, wherever it sits; a compact `rocm102` is the digits of that +/// segment alone, so `rocm102` as the whole local version and `d20261110.rocm102` after +/// other segments are both read. +fn local_rocm_matches(local: &str, rocm_sdk_version: &str) -> Option { + let segments: Vec<&str> = local.split('.').collect(); + let at = segments + .iter() + .position(|segment| segment.starts_with("rocm"))?; + let dotted = segments[at..].join("."); + if let Some(recorded) = dotted.strip_prefix("rocm") + && rocm_series(recorded).is_some() + { + return Some(rocm_sdk_series_matches(recorded, rocm_sdk_version)); + } + // `.rocm102` names 10.2 with no dot between major and minor. A `rocm` segment + // that is neither spelling (`+rocm`, `+rocmabc`) names no line this SDK can be + // matched to, so it is a mismatch, not a pass. + Some(compact_rocm_local_segment(local).is_some_and(|compact| { + rocm_series(rocm_sdk_version) + .is_some_and(|(major, minor)| compact == format!("{major}{minor}")) + })) +} +/// The digits of a `rocm` segment in a dotted local version, e.g. +/// `102` for `g5df63a720.d20261110.rocm102`: the spelling the 10.2 vLLM wheels +/// use, with no dot between the ROCm major and minor. +fn compact_rocm_local_segment(local: &str) -> Option<&str> { + local + .split('.') + .filter_map(|segment| segment.strip_prefix("rocm")) + .find(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())) +} /// The PEP 425 interpreter tag of `python`, e.g. `cp314`. fn python_interpreter_tag(python: &Path) -> Result { let output = ProcessCommand::new(python) @@ -666,37 +1111,37 @@ fn install_vllm_rocm10_discover( reinstall: bool, build: &VllmRocmDiscoverBuild, ) -> Result { + let mut notes: Vec = Vec::new(); + // Every failure after a fallback was taken names it, wherever it happens: a note + // kept only for the success path would be lost exactly when the fallback it + // describes is what failed. + match install_vllm_rocm10_discover_collecting(uv, paths, python, reinstall, build, &mut notes) { + Ok(pins) => Ok(VllmInstallOutcome { pins, notes }), + Err(err) => Err(with_fallback_notes(err, ¬es)), + } +} +/// [`install_vllm_rocm10_discover`]'s sequence, recording each fallback it takes in +/// `notes` as it goes and returning the pins installed. +fn install_vllm_rocm10_discover_collecting( + uv: &Path, + paths: &AppPaths, + python: &Path, + reinstall: bool, + build: &VllmRocmDiscoverBuild, + notes: &mut Vec, +) -> Result> { ensure_discover_python_tag(python, build)?; - let torch = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torch", - DiscoverVersion::Series(build.torch_version_prefix), - )?; - // Discovered fresh from the same index and row as torch, rather than - // snapshotted from whatever the SDK install resolved earlier: that - // snapshot can pair with a *different* torch version than the one this - // row pins (the SDK's own install is unpinned and may have picked the - // newest torch/torchvision/torchaudio available, independently of each - // other), producing an ABI-incompatible triplet at realign time. - let torchvision = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torchvision", - DiscoverVersion::Series(build.torchvision_version_prefix), - )?; - let torchaudio = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torchaudio", - build.torchaudio_version, - )?; + // Discovered (or read from the SDK runtime) together, as one coherent set, + // rather than snapshotted from whatever the SDK install resolved earlier + // for a row that discovers: that snapshot can pair with a *different* + // torch version than the one this row pins, producing an ABI-incompatible + // triplet at realign time. + let mut torch_stack = torch_stack_pins(uv, paths, python, build)?.into_iter(); + let (torch, torchvision, torchaudio) = ( + torch_stack.next().context("torch pin")?, + torch_stack.next().context("torchvision pin")?, + torch_stack.next().context("torchaudio pin")?, + ); let vllm = discover_pinned_requirement( uv, paths, @@ -710,7 +1155,6 @@ fn install_vllm_rocm10_discover( // not fail the install: the real install below is the authority, and it // reports its own error. The reason is kept, because the series match is // exactly the choice this probe exists to improve on. - let mut notes: Vec = Vec::new(); let vllm_resolution = match resolve_vllm_dependencies(uv, paths, python, &vllm, &config_file) { Ok(resolution) => Some(resolution), Err(err) => { @@ -751,34 +1195,42 @@ fn install_vllm_rocm10_discover( // Two separate `uv pip install` calls, each scoped to exactly one index: // see `vllm_rocm10_discover_torch_install_args` for why combining both // indexes into one call is unsafe. - let torch_args = vllm_rocm10_discover_torch_install_args(python, reinstall, build, &pins[0]); + let torch_index_url = if build.torch_stack == TorchStack::FromInstalledSdk { + let recorded_index = recorded_sdk_index_url(paths, python); + match &recorded_index { + Ok(Some(_)) => {} + Ok(None) => notes.push(format!( + "no runtime manifest for this interpreter records a wheel index; the torch stack is \ + fetched from {}, which may not be the index the SDK was installed from", + build.torch_index_url + )), + Err(err) => notes.push(format!( + "could not read the runtime registry ({err:#}); the torch stack is fetched \ + from {}, which may not be the index the SDK was installed from", + build.torch_index_url + )), + } + torch_index_url_for(build, recorded_index.ok().flatten().as_deref()) + } else { + build.torch_index_url.to_owned() + }; + let torch_args = + vllm_rocm10_discover_torch_install_args(python, reinstall, &torch_index_url, &pins[0]); run_uv_pip_install(uv, paths, python, torch_args)?; let mut remaining_args = vllm_rocm10_discover_remaining_install_args(python, reinstall, &pins[3..]); remaining_args.push("--config-file".to_owned()); remaining_args.push(config_file.display().to_string()); - if let Err(err) = run_uv_pip_install(uv, paths, python, remaining_args) { - // A note kept only for the success path would be lost exactly when the - // fallback pins are what failed. - return Err(if notes.is_empty() { - err - } else { - // The cause first, so plain `{}` display leads with the uv failure - // and not with a note about how the pins were chosen. - anyhow!( - "{err:#}\n\nThe fallback pins were used because: {}", - notes.join(" ") - ) - }); - } + run_uv_pip_install(uv, paths, python, remaining_args)?; // The full-dependency resolve above can replace torch (and, pulling it in // transitively, torchvision/torchaudio) with unconstrained PyPI builds; // force the whole stack back to the coherent trio discovered above. This // must run unconditionally, before the device-plugin discovery below: a // `uv pip freeze` failure there must not skip restoring the trio too. - let trio_realign_args = vllm_rocm10_discover_realign_install_args(python, build, &pins[..3]); + let trio_realign_args = + vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &pins[..3]); run_uv_pip_install(uv, paths, python, trio_realign_args)?; // The full-dependency resolve can also pull in an AMD device-kernel @@ -807,114 +1259,674 @@ fn install_vllm_rocm10_discover( } if !device_pins.is_empty() { let device_realign_args = - vllm_rocm10_discover_realign_install_args(python, build, &device_pins); + vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &device_pins); run_uv_pip_install(uv, paths, python, device_realign_args)?; } let mut pins = pins; pins.extend(device_pins); - Ok(VllmInstallOutcome { pins, notes }) + Ok(pins) } #[cfg(test)] mod tests { use super::*; + const KEY_ERROR_UNIX: &str = "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/venv/lib/python3.14/site-packages/tensorizer/serialization.py\", line 40, in \n from . import _crypt\n File \"/venv/lib/python3.14/site-packages/tensorizer/_crypt/_encryption.py\", line 367, in __new__\n type, NamedTuple(name, **{k: annotations[k] for k in entries})\nKeyError: 'crypto_stream_salsa20_NONCEBYTES'\n"; + #[test] - fn dry_run_resolved_pin_parses_a_rotated_dev_tag() { - let stdout = "Resolved 1 package in 601ms\nWould download 1 package\nWould install 1 package\n + vllm==0.27.1.dev5+rocm10.0.0.gf46a9dfe2.d20260826\n"; + fn the_python_314_key_error_is_a_broken_tensorizer() { assert_eq!( - dry_run_resolved_pin(stdout, "vllm"), - Some("vllm==0.27.1.dev5+rocm10.0.0.gf46a9dfe2.d20260826".to_owned()) + classify_tensorizer_failure(KEY_ERROR_UNIX), + TensorizerProbe::Broken("KeyError: 'crypto_stream_salsa20_NONCEBYTES'".to_owned()) ); } + #[test] - fn dry_run_resolved_pin_parses_a_simple_version() { - let stdout = "Resolved 1 package in 553ms\n + flash-attn==2.8.3\n"; - assert_eq!( - dry_run_resolved_pin(stdout, "flash-attn"), - Some("flash-attn==2.8.3".to_owned()) - ); + fn a_windows_style_tensorizer_path_is_recognised() { + let stderr = "Traceback (most recent call last):\n File \"C:\\venv\\Lib\\site-packages\\tensorizer\\_crypt\\_encryption.py\", line 367, in __new__\nKeyError: 'x'\n"; + assert!(matches!( + classify_tensorizer_failure(stderr), + TensorizerProbe::Broken(_) + )); } + #[test] - fn dry_run_resolved_pin_ignores_other_packages_and_missing_lines() { - let stdout = "Resolved 1 package in 553ms\n + amd-aiter==0.1.20.post1\n"; - assert_eq!(dry_run_resolved_pin(stdout, "vllm"), None); - assert_eq!(dry_run_resolved_pin("no solution found", "vllm"), None); + fn a_torch_failure_with_tensorizer_frames_is_not_a_broken_tensorizer() { + // `import tensorizer.serialization` imports torch, so a missing HIP library + // has tensorizer frames above the torch ones. + let stderr = "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/venv/lib/python3.14/site-packages/tensorizer/serialization.py\", line 12, in \n import torch\n File \"/venv/lib/python3.14/site-packages/torch/__init__.py\", line 300, in \n _load_global_deps()\nOSError: libamdhip64.so.7: cannot open shared object file: No such file or directory\n"; + assert_eq!(classify_tensorizer_failure(stderr), TensorizerProbe::Usable); } + #[test] - fn vllm_rocm10_discover_torch_install_args_use_only_the_torch_index() { - let python = PathBuf::from("/opt/venv/bin/python"); - let build = vllm_rocm_discover_build("10.0.0").expect("10.0.0 has a discover row"); - let torch_pin = "torch==2.12.0+rocm10.0.0"; + fn a_non_load_exception_raised_below_tensorizer_is_not_a_broken_tensorizer() { + // Tensorizer frames above, but the exception comes from torch and is not a + // library-load error, so only the last-frame rule can acquit tensorizer. + let stderr = "Traceback (most recent call last):\n File \"\", line 1, in \n File \"/venv/lib/python3.14/site-packages/tensorizer/serialization.py\", line 12, in \n import torch\n File \"/venv/lib/python3.14/site-packages/torch/_utils.py\", line 80, in helper\n return table[key]\nKeyError: 'gfx1151'\n"; + assert_eq!(classify_tensorizer_failure(stderr), TensorizerProbe::Usable); + } - let args = vllm_rocm10_discover_torch_install_args(&python, false, build, torch_pin); - assert!(!args.contains(&"--reinstall".to_owned())); - assert!(!args.contains(&"--reinstall-package".to_owned())); - assert!(args.contains(&torch_pin.to_owned())); - assert!(args.contains(&"--no-deps".to_owned())); - assert!(args.contains(&"--prerelease".to_owned())); - assert!(args.contains(&"allow".to_owned())); - assert!(args.contains(&build.torch_index_url.to_owned())); - assert!( - args.contains(&"--index-url".to_owned()), - "{args:?} must use --index-url, not --extra-index-url, or a same-numbered public \ - PyPI wheel can silently outrank AMD's build" - ); - assert!(!args.contains(&"--extra-index-url".to_owned())); - assert!( - !args.contains(&build.vllm_index_url.to_owned()), - "{args:?} must never reference vllm_index_url, or `uv` may probe torch under it \ - and hit a fatal 403 from a nonexistent sub-path" - ); + /// Each case raises its exception with tensorizer as the LAST frame, so only + /// the library-load rule can acquit it, and each carries exactly one of the + /// rule's markers: the exception type or the message. + #[test] + fn each_library_load_marker_alone_acquits_a_tensorizer_frame() { + let cases = [ + ( + "OSError type alone", + "OSError: [Errno 2] No such file or directory: 'libsodium.so'", + ), + ( + "ImportError type alone", + "ImportError: cannot import name 'x' from 'y'", + ), + ( + "ModuleNotFoundError type alone", + "ModuleNotFoundError: No module named 'redis'", + ), + ( + "'shared object file' alone, wrapped in another type", + "RuntimeError: libamdhip64.so.7: cannot open shared object file", + ), + ( + "'DLL load failed' alone, wrapped in another type", + "RuntimeError: DLL load failed while importing _tensorizer: The specified module could not be found.", + ), + ]; + for (what, exception) in cases { + let stderr = format!( + "Traceback (most recent call last):\n File \"/venv/site-packages/tensorizer/serialization.py\", line 3, in \n{exception}\n" + ); + assert_eq!( + classify_tensorizer_failure(&stderr), + TensorizerProbe::Usable, + "{what}" + ); + } + // The same frame with a non-load exception is tensorizer's own failure. + let broken = "Traceback (most recent call last):\n File \"/venv/site-packages/tensorizer/serialization.py\", line 3, in \nKeyError: 'x'\n"; + assert!(matches!( + classify_tensorizer_failure(broken), + TensorizerProbe::Broken(_) + )); + } - let args = vllm_rocm10_discover_torch_install_args(&python, true, build, torch_pin); - assert!(args.contains(&"--reinstall-package".to_owned())); - assert!(args.contains(&"torch".to_owned())); - assert!(!args.contains(&"--reinstall".to_owned())); + #[test] + fn an_absent_tensorizer_or_empty_stderr_is_not_a_finding() { + let absent = "Traceback (most recent call last):\n File \"\", line 1, in \nModuleNotFoundError: No module named 'tensorizer'\n"; + assert_eq!(classify_tensorizer_failure(absent), TensorizerProbe::Usable); + assert_eq!(classify_tensorizer_failure(""), TensorizerProbe::Usable); + } + + fn broken() -> TensorizerProbe { + TensorizerProbe::Broken("KeyError".to_owned()) } + #[test] - fn vllm_rocm10_discover_remaining_install_args_use_only_the_vllm_index() { - let pins = vec![ - "vllm==0.27.1.dev5+rocm10.0.0".to_owned(), - "flash-attn==2.8.3".to_owned(), - "amd-aiter==0.1.4".to_owned(), - "tensorizer==2.12.1".to_owned(), - ]; - let python = PathBuf::from("/opt/venv/bin/python"); - let build = vllm_rocm_discover_build("10.0.0").expect("10.0.0 has a discover row"); + fn an_unmanaged_runtime_is_never_probed_or_changed() -> Result<()> { + let note = tensorizer_step( + false, + true, + || panic!("an environment rocm-cli does not own must not be probed"), + || panic!("an environment rocm-cli does not own must not be changed"), + )?; + assert_eq!(note, None); + let runtime = VllmRuntime { + source: "environment python".to_owned(), + python_executable: Some(PathBuf::from("/nonexistent/python")), + ..VllmRuntime::for_test() + }; + assert!(!runtime_is_managed(&runtime)); + assert_eq!(ensure_runtime_tensorizer(&runtime, true)?, None); + Ok(()) + } - let args = vllm_rocm10_discover_remaining_install_args(&python, false, &pins); - assert!(!args.contains(&"--reinstall-package".to_owned())); - for pin in &pins { - assert!(args.contains(pin), "{args:?} should contain {pin}"); - } - assert!(args.contains(&"--prerelease".to_owned())); - assert!(args.contains(&"allow".to_owned())); + #[test] + fn a_usable_tensorizer_is_left_alone() -> Result<()> { + let note = tensorizer_step( + true, + true, + || Ok(TensorizerProbe::Usable), + || panic!("a usable tensorizer must not be upgraded"), + )?; + assert_eq!(note, None); + Ok(()) + } + + #[test] + fn a_broken_tensorizer_is_upgraded_once_and_probed_again() -> Result<()> { + let mut probes = vec![TensorizerProbe::Usable, broken()]; + let mut upgrades = 0; + let note = tensorizer_step( + true, + true, + || Ok(probes.pop().expect("probed more often than expected")), + || { + upgrades += 1; + Ok(()) + }, + )? + .expect("an override is reported"); + assert_eq!(upgrades, 1); + assert!(probes.is_empty(), "the upgrade must be probed again"); assert!( - !args.contains(&build.vllm_index_url.to_owned()), - "{args:?} must supply vllm_index_url via --config-file, not --extra-index-url, or \ - `uv` 403s fatally on a package the index doesn't host despite ignore-error-codes" + note.contains("2.12.1") && note.contains("on purpose"), + "{note}" ); + Ok(()) + } + + #[test] + fn an_upgrade_that_does_not_help_fails_an_install_this_run_made_and_only_warns_otherwise() + -> Result<()> { + let note = tensorizer_step(true, false, || Ok(broken()), || Ok(()))? + .expect("a runtime this run did not install into gets a note"); + assert!(note.contains("still cannot be imported"), "{note}"); assert!( - !args.contains(&build.torch_index_url.to_owned()), - "{args:?} must never reference torch_index_url, or `uv` may probe flash-attn/\ - amd-aiter under it and hit a fatal 403 from a nonexistent sub-path" + !note.contains("was installed"), + "no install ran on that runtime: {note}" + ); + let error = tensorizer_step(true, true, || Ok(broken()), || Ok(())) + .expect_err("an install whose tensorizer still cannot import must not succeed"); + assert!( + format!("{error:#}").contains("vLLM was installed, but tensorizer still cannot"), + "{error:#}" ); + Ok(()) + } - let args = vllm_rocm10_discover_remaining_install_args(&python, true, &pins); - for pin in &pins { - let name = pin.split_once("==").unwrap().0; - assert!(args.contains(&"--reinstall-package".to_owned())); - assert!( - args.contains(&name.to_owned()), - "{args:?} should reinstall {name}" - ); - } - assert!(!args.contains(&"--reinstall".to_owned())); + #[test] + fn a_failed_upgrade_fails_an_install_but_only_warns_on_an_existing_runtime() -> Result<()> { + let fail = || -> Result<()> { bail!("network unreachable") }; + let error = tensorizer_step(true, true, || Ok(broken()), fail) + .expect_err("an install must not leave an unusable runtime"); + // The message already carries the cause once; chaining the error as a + // source as well would print it twice under `{:#}`. + assert_eq!( + format!("{error:#}").matches("network unreachable").count(), + 1, + "{error:#}" + ); + let note = tensorizer_step(true, false, || Ok(broken()), fail)? + .expect("the failure is reported as a note"); + assert!( + note.contains("network unreachable") && note.contains("would fail at startup"), + "{note}" + ); + Ok(()) } - /// A stand-in `uv` that answers the two kinds of dry-run the discovery makes the - /// way AMD's staging index did when ROCm 10.1's vLLM stopped installing: the - /// index serves a newer `amd-aiter` (0.1.24.post1) than the one the newest - /// vLLM 0.29 wheel declares (a 0.1.22 dev build), and the `--no-deps` + + #[cfg(unix)] + #[test] + fn the_probe_reads_the_interpreters_traceback() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let python = root.path().join("python"); + write_fake_executable( + &python, + format!("#!/bin/sh\ncat >&2 <<'EOF'\n{KEY_ERROR_UNIX}EOF\nexit 1\n"), + )?; + assert!(matches!( + probe_tensorizer_import(&python, None)?, + TensorizerProbe::Broken(_) + )); + Ok(()) + } + + /// A ROCm 10.x discover runtime whose stand-in interpreter, like a real one, + /// does not see `PYTHONPATH` under `-I` (which implies `-E`), and without it + /// stops on a missing module before it ever reaches tensorizer. + #[cfg(unix)] + #[test] + fn the_probe_keeps_the_pythonpath_the_server_launches_with() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let venv = tempfile::tempdir()?; + std::fs::create_dir_all(venv.path().join("bin"))?; + std::fs::create_dir_all(venv.path().join("lib/python3.14/site-packages"))?; + let python = venv.path().join("bin/python"); + write_fake_executable( + &python, + format!( + "#!/bin/sh\ncase \" $* \" in *\" -I \"*) unset PYTHONPATH ;; esac\nif [ -z \"$PYTHONPATH\" ]; then\n echo \"ModuleNotFoundError: No module named 'amdsmi'\" >&2\n exit 1\nfi\ncat >&2 <<'EOF'\n{KEY_ERROR_UNIX}EOF\nexit 1\n" + ), + )?; + let runtime = VllmRuntime { + python_executable: Some(python), + rocm_sdk_version: Some("10.1.0".to_owned()), + ..VllmRuntime::for_test() + }; + assert!( + matches!( + probe_tensorizer_import( + runtime.python_executable.as_deref().unwrap(), + Some(&runtime) + )?, + TensorizerProbe::Broken(_) + ), + "a missing module the server's PYTHONPATH provides must not hide a broken tensorizer" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn the_upgrade_is_a_no_deps_install_of_the_fixed_release() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let paths = AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let log = root.path().join("uv.log"); + let uv = root.path().join("uv"); + write_fake_executable( + &uv, + format!("#!/bin/sh\necho \"$@\" >> \"{}\"\n", log.display()), + )?; + upgrade_tensorizer(&uv, &paths, Path::new("/venv/bin/python"))?; + let calls = std::fs::read_to_string(&log)?; + assert!( + calls.contains("--no-deps") && calls.contains("tensorizer==2.12.1"), + "{calls}" + ); + Ok(()) + } + + #[test] + fn a_coloured_traceback_is_still_a_broken_tensorizer() { + // What Python 3.13+ writes under FORCE_COLOR: escapes wrap the quoted path. + let coloured = "\u{1b}[1;35mTraceback (most recent call last):\u{1b}[0m\n File \u{1b}[35m\"/venv/lib/python3.14/site-packages/tensorizer/_crypt/_encryption.py\u{1b}[0m\", line \u{1b}[35m367\u{1b}[0m, in \u{1b}[35m__new__\u{1b}[0m\n\u{1b}[1;35mKeyError\u{1b}[0m: \u{1b}[35m'crypto_stream_salsa20_NONCEBYTES'\u{1b}[0m\n"; + assert_eq!( + classify_tensorizer_failure(coloured), + TensorizerProbe::Broken("KeyError: 'crypto_stream_salsa20_NONCEBYTES'".to_owned()) + ); + } + + #[test] + fn the_probe_asks_for_no_colour_and_runs_in_the_launch_environment() { + let runtime = VllmRuntime::for_test(); + let command = tensorizer_probe_command(Path::new("python"), Some(&runtime)) + .expect("a bare runtime has a launch environment"); + let env = |key: &str| { + command + .get_envs() + .find(|(k, _)| *k == std::ffi::OsStr::new(key)) + .map(|(_, v)| v.map(std::ffi::OsStr::to_owned)) + }; + assert_eq!( + env("FORCE_COLOR"), + Some(None), + "FORCE_COLOR must be removed" + ); + assert_eq!(env("NO_COLOR"), Some(Some("1".into()))); + let args: Vec<_> = command.get_args().collect(); + assert!( + !args.contains(&std::ffi::OsStr::new("-I")) + && !args.contains(&std::ffi::OsStr::new("-E")), + "-I and -E drop the PYTHON* environment the server launches with: {args:?}" + ); + assert!( + args.contains(&std::ffi::OsStr::new("-s")), + "the user's site directory must stay out of the probe: {args:?}" + ); + assert_eq!( + env("PYTHONSAFEPATH"), + Some(Some("1".into())), + "the working directory must stay off sys.path" + ); + assert_eq!(env("PYTHON_COLORS"), Some(Some("0".into()))); + assert_eq!( + env("VLLM_TARGET_DEVICE"), + Some(Some("rocm".into())), + "the probe must carry the environment the server launches with" + ); + let bare = + tensorizer_probe_command(Path::new("python"), None).expect("no runtime to build"); + assert!( + bare.get_envs() + .all(|(k, _)| k != std::ffi::OsStr::new("VLLM_TARGET_DEVICE")) + ); + } + + #[test] + fn a_probe_that_cannot_run_fails_an_install_but_only_warns_on_an_existing_runtime() -> Result<()> + { + let cannot = || -> Result { bail!("timed out after 300s") }; + let error = tensorizer_step(true, true, cannot, || Ok(())) + .expect_err("an install must not report a runtime it could not check"); + assert!(format!("{error:#}").contains("timed out")); + let note = tensorizer_step(true, false, cannot, || Ok(()))? + .expect("an existing runtime gets a note"); + assert!( + note.contains("could not check") && note.contains("timed out"), + "{note}" + ); + + // The second probe, after a successful upgrade, is held to the same rule. + let mut calls = 0; + let second_fails = || { + calls += 1; + if calls == 1 { + Ok(broken()) + } else { + bail!("spawn failed") + } + }; + let note = tensorizer_step(true, false, second_fails, || Ok(()))? + .expect("an existing runtime gets a note"); + assert!(note.contains("could not check"), "{note}"); + Ok(()) + } + + /// A managed runtime whose interpreter is a stand-in that raises the Python + /// 3.14 `KeyError` only when launched with the server's environment, and a + /// torch-style `OSError` otherwise. + #[cfg(unix)] + fn runtime_needing_launch_env(root: &Path) -> Result { + use rocm_core::fake_executable::write_fake_executable; + let python = root.join("python"); + write_fake_executable( + &python, + format!( + "#!/bin/sh\nif [ \"$VLLM_TARGET_DEVICE\" = rocm ]; then\ncat >&2 <<'EOF'\n{KEY_ERROR_UNIX}EOF\nelse\necho 'OSError: libamdhip64.so.7: cannot open shared object file' >&2\nfi\nexit 1\n" + ), + )?; + Ok(VllmRuntime { + python_executable: Some(python), + ..VllmRuntime::for_test() + }) + } + + #[cfg(unix)] + #[test] + fn the_probe_sees_the_launch_environment_and_the_upgrade_follows() -> Result<()> { + let root = tempfile::tempdir()?; + let runtime = runtime_needing_launch_env(root.path())?; + let mut upgraded = false; + // Still broken after the "upgrade": the stand-in never changes, so the + // refusal proves the probe ran with the launch environment both times. + let error = ensure_runtime_tensorizer_with(&runtime, true, |_| { + upgraded = true; + Ok(()) + }) + .expect_err("the stand-in stays broken"); + assert!( + upgraded, + "the Python 3.14 failure was only visible with the launch environment" + ); + assert!(format!("{error:#}").contains("still cannot be imported")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_upgrade_fails_an_install_this_run_made_and_only_warns_on_an_existing_runtime() + -> Result<()> { + let root = tempfile::tempdir()?; + let runtime = runtime_needing_launch_env(root.path())?; + let fail = |_: &Path| -> Result<()> { bail!("network unreachable") }; + let error = ensure_runtime_tensorizer_with(&runtime, true, fail) + .expect_err("an install this run made must fail"); + assert!(format!("{error:#}").contains("network unreachable")); + let note = ensure_runtime_tensorizer_with(&runtime, false, fail)? + .expect("an existing runtime only gets a note"); + assert!(note.contains("network unreachable"), "{note}"); + Ok(()) + } + + #[cfg(target_os = "linux")] + #[test] + fn a_hung_probe_is_abandoned_promptly_and_its_child_killed() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let pid_file = root.path().join("pid"); + let python = root.path().join("python"); + write_fake_executable( + &python, + format!( + "#!/bin/sh\necho $$ > \"{}\"\nexec sleep 30\n", + pid_file.display() + ), + )?; + let started = std::time::Instant::now(); + let error = + probe_tensorizer_import_within(&python, None, std::time::Duration::from_millis(300)) + .expect_err("a hung probe must be abandoned"); + assert!(format!("{error:#}").contains("exceeded"), "{error:#}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "the probe waited out the child: {:?}", + started.elapsed() + ); + let pid = std::fs::read_to_string(&pid_file)?; + assert!( + !Path::new(&format!("/proc/{}", pid.trim())).exists(), + "the hung child was left running" + ); + Ok(()) + } + + /// 5 MB on stderr before the real traceback: more than a pipe holds, so an + /// undrained pipe would stall the child until the deadline. + #[cfg(unix)] + #[test] + fn a_probe_that_prints_more_than_a_pipe_holds_is_still_classified() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let python = root.path().join("python"); + write_fake_executable( + &python, + format!( + "#!/bin/sh\nhead -c 5000000 /dev/zero | tr '\\0' 'x' >&2\necho >&2\ncat >&2 <<'EOF'\n{KEY_ERROR_UNIX}EOF\nexit 1\n" + ), + )?; + let probe = + probe_tensorizer_import_within(&python, None, std::time::Duration::from_secs(20))?; + assert!(matches!(probe, TensorizerProbe::Broken(_)), "{probe:?}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_upgrade_that_prints_more_than_a_pipe_holds_still_finishes() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let paths = AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let uv = root.path().join("uv"); + write_fake_executable(&uv, "#!/bin/sh\nhead -c 5000000 /dev/zero >&2\nexit 0\n")?; + upgrade_tensorizer_within( + &uv, + &paths, + Path::new("/venv/bin/python"), + std::time::Duration::from_secs(20), + ) + } + + #[cfg(target_os = "linux")] + #[test] + fn a_hung_upgrade_is_abandoned_promptly_and_its_child_killed() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let paths = AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let pid_file = root.path().join("pid"); + let uv = root.path().join("uv"); + write_fake_executable( + &uv, + format!( + "#!/bin/sh\necho $$ > \"{}\"\nexec sleep 30\n", + pid_file.display() + ), + )?; + let started = std::time::Instant::now(); + let error = upgrade_tensorizer_within( + &uv, + &paths, + Path::new("/venv/bin/python"), + std::time::Duration::from_millis(300), + ) + .expect_err("a hung upgrade must be abandoned"); + assert!(format!("{error:#}").contains("exceeded"), "{error:#}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "the upgrade waited out the child: {:?}", + started.elapsed() + ); + let pid = std::fs::read_to_string(&pid_file)?; + assert!( + !Path::new(&format!("/proc/{}", pid.trim())).exists(), + "the hung child was left running" + ); + Ok(()) + } + + #[test] + fn dry_run_resolved_pin_parses_a_rotated_dev_tag() { + let stdout = "Resolved 1 package in 601ms\nWould download 1 package\nWould install 1 package\n + vllm==0.27.1.dev5+rocm10.0.0.gf46a9dfe2.d20260826\n"; + assert_eq!( + dry_run_resolved_pin(stdout, "vllm"), + Some("vllm==0.27.1.dev5+rocm10.0.0.gf46a9dfe2.d20260826".to_owned()) + ); + } + #[test] + fn dry_run_resolved_pin_parses_a_simple_version() { + let stdout = "Resolved 1 package in 553ms\n + flash-attn==2.8.3\n"; + assert_eq!( + dry_run_resolved_pin(stdout, "flash-attn"), + Some("flash-attn==2.8.3".to_owned()) + ); + } + #[test] + fn dry_run_resolved_pin_ignores_other_packages_and_missing_lines() { + let stdout = "Resolved 1 package in 553ms\n + amd-aiter==0.1.20.post1\n"; + assert_eq!(dry_run_resolved_pin(stdout, "vllm"), None); + assert_eq!(dry_run_resolved_pin("no solution found", "vllm"), None); + } + #[test] + fn vllm_rocm10_discover_torch_install_args_use_only_the_torch_index() { + let python = PathBuf::from("/opt/venv/bin/python"); + let build = vllm_rocm_discover_build("10.0.0").expect("10.0.0 has a discover row"); + let torch_pin = "torch==2.12.0+rocm10.0.0"; + + let args = vllm_rocm10_discover_torch_install_args( + &python, + false, + build.torch_index_url, + torch_pin, + ); + assert!(!args.contains(&"--reinstall".to_owned())); + assert!(!args.contains(&"--reinstall-package".to_owned())); + assert!(args.contains(&torch_pin.to_owned())); + assert!(args.contains(&"--no-deps".to_owned())); + assert!(args.contains(&"--prerelease".to_owned())); + assert!(args.contains(&"allow".to_owned())); + assert!(args.contains(&build.torch_index_url.to_owned())); + assert!( + args.contains(&"--index-url".to_owned()), + "{args:?} must use --index-url, not --extra-index-url, or a same-numbered public \ + PyPI wheel can silently outrank AMD's build" + ); + assert!(!args.contains(&"--extra-index-url".to_owned())); + assert!( + !args.contains(&build.vllm_index_url.to_owned()), + "{args:?} must never reference vllm_index_url, or `uv` may probe torch under it \ + and hit a fatal 403 from a nonexistent sub-path" + ); + + let args = vllm_rocm10_discover_torch_install_args( + &python, + true, + build.torch_index_url, + torch_pin, + ); + assert!(args.contains(&"--reinstall-package".to_owned())); + assert!(args.contains(&"torch".to_owned())); + assert!(!args.contains(&"--reinstall".to_owned())); + } + #[test] + fn an_installed_sdk_row_fetches_its_torch_stack_from_the_index_the_sdk_came_from() { + let python = PathBuf::from("/opt/venv/bin/python"); + let release = "https://stable.repo.amd.com/rocm/whl-next"; + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 has a discover row"); + assert_eq!(ten_two.torch_stack, TorchStack::FromInstalledSdk); + + // A release SDK's torch is not on the row's nightly index, so the row's own + // index must not be the one handed to uv. + let index = torch_index_url_for(ten_two, Some(release)); + assert_eq!(index, release); + assert_ne!(index, ten_two.torch_index_url); + let pins = vec!["torch==2.14.0+rocm10.2.0".to_owned()]; + let install = vllm_rocm10_discover_torch_install_args(&python, false, &index, &pins[0]); + let realign = vllm_rocm10_discover_realign_install_args(&python, &index, &pins); + for args in [&install, &realign] { + assert!(args.contains(&release.to_owned()), "{args:?}"); + assert!( + !args.contains(&ten_two.torch_index_url.to_owned()), + "{args:?}" + ); + } + + // A manifest that recorded no index keeps the row's. + assert_eq!(torch_index_url_for(ten_two, None), ten_two.torch_index_url); + // A row that discovers its stack never takes the recorded index. + let ten_one = vllm_rocm_discover_build("10.1.0").expect("10.1 has a discover row"); + assert_eq!(ten_one.torch_stack, TorchStack::Discover); + assert_eq!( + torch_index_url_for(ten_one, Some(release)), + ten_one.torch_index_url + ); + } + #[test] + fn vllm_rocm10_discover_remaining_install_args_use_only_the_vllm_index() { + let pins = vec![ + "vllm==0.27.1.dev5+rocm10.0.0".to_owned(), + "flash-attn==2.8.3".to_owned(), + "amd-aiter==0.1.4".to_owned(), + "tensorizer==2.12.1".to_owned(), + ]; + let python = PathBuf::from("/opt/venv/bin/python"); + let build = vllm_rocm_discover_build("10.0.0").expect("10.0.0 has a discover row"); + + let args = vllm_rocm10_discover_remaining_install_args(&python, false, &pins); + assert!(!args.contains(&"--reinstall-package".to_owned())); + for pin in &pins { + assert!(args.contains(pin), "{args:?} should contain {pin}"); + } + assert!(args.contains(&"--prerelease".to_owned())); + assert!(args.contains(&"allow".to_owned())); + assert!( + !args.contains(&build.vllm_index_url.to_owned()), + "{args:?} must supply vllm_index_url via --config-file, not --extra-index-url, or \ + `uv` 403s fatally on a package the index doesn't host despite ignore-error-codes" + ); + assert!( + !args.contains(&build.torch_index_url.to_owned()), + "{args:?} must never reference torch_index_url, or `uv` may probe flash-attn/\ + amd-aiter under it and hit a fatal 403 from a nonexistent sub-path" + ); + + let args = vllm_rocm10_discover_remaining_install_args(&python, true, &pins); + for pin in &pins { + let name = pin.split_once("==").unwrap().0; + assert!(args.contains(&"--reinstall-package".to_owned())); + assert!( + args.contains(&name.to_owned()), + "{args:?} should reinstall {name}" + ); + } + assert!(!args.contains(&"--reinstall".to_owned())); + } + /// A stand-in `uv` that answers the two kinds of dry-run the discovery makes the + /// way AMD's staging index did when ROCm 10.1's vLLM stopped installing: the + /// index serves a newer `amd-aiter` (0.1.24.post1) than the one the newest + /// vLLM 0.29 wheel declares (a 0.1.22 dev build), and the `--no-deps` /// single-index query serves only builds that index really holds. #[cfg(unix)] fn fake_uv_for_staging_index(dir: &Path) -> PathBuf { @@ -1118,6 +2130,248 @@ exit 0 Ok((outcome, calls)) } + /// Runs `install_vllm_rocm10_discover` on the 10.2 row against a fake `uv` + /// whose `pip freeze` reports a release SDK's torch stack, with `prepare` + /// free to set up the runtime registry first. Returns the outcome and every + /// `uv` argv. + #[cfg(unix)] + fn run_ten_two_discover_with_fake_uv( + prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, + ) -> Result<(Result, Vec)> { + run_discover_row_with_fake_uv("10.2.0", None, prepare) + } + /// [`run_ten_two_discover_with_fake_uv`] for any row, optionally failing the + /// `fail_install`th real `uv pip install` (1 torch, 2 remaining deps, 3 trio + /// realign, 4 device realign; the dry-runs and `pip freeze` are not counted). + #[cfg(unix)] + fn run_discover_row_with_fake_uv( + row: &str, + fail_install: Option, + prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, + ) -> Result<(Result, Vec)> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let paths = AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let python = root.path().join("python"); + write_fake_executable(&python, "#!/bin/sh\necho cp314\n")?; + prepare(&paths, &python)?; + let log = root.path().join("uv.log"); + let uv = root.path().join("uv"); + write_fake_executable( + &uv, + format!( + r#"#!/bin/sh +echo "$@" >> "{log}" +last="" +for a in "$@"; do last="$a"; done +case "$*" in + *--dry-run*|*"pip freeze"*) ;; + *) + n=$(cat "{count}" 2>/dev/null || echo 0) + n=$((n + 1)) + echo "$n" > "{count}" + if [ "$n" = "{fail_at}" ]; then echo 'error: boom' >&2; exit 1; fi + ;; +esac +case "$*" in + *--dry-run*--config-file*) + echo "Resolved 3 packages" >&2 + ;; + *--dry-run*) + pkg=${{last%%==*}} + case "$last" in + *'*'*) echo " + ${{pkg}}==9.9.9" >&2 ;; + *) echo " + ${{last}}" >&2 ;; + esac + ;; + *"pip freeze"*) + echo "torch==2.14.0+rocm10.2.0" + echo "torchvision==0.29.2+rocm10.2.0" + echo "torchaudio==2.11.1+rocm10.2.0" + echo "amd-torch-device-gfx1151==2.14.0+rocm10.2.0" + ;; +esac +exit 0 +"#, + log = log.display(), + count = root.path().join("uv.count").display(), + fail_at = fail_install.unwrap_or(0) + ), + )?; + let build = vllm_rocm_discover_build(row).expect("the row has a discover build"); + let outcome = install_vllm_rocm10_discover(&uv, &paths, &python, true, build); + let calls = std::fs::read_to_string(&log)? + .lines() + .map(str::to_owned) + .collect(); + Ok((outcome, calls)) + } + + #[cfg(unix)] + #[test] + fn a_release_sdk_on_the_ten_two_row_gets_its_torch_stack_from_its_own_index() -> Result<()> { + let release = "https://stable.example/rocm/whl-next"; + let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, python| { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::write( + registry.join("release.json"), + format!( + r#"{{"python_executable":"{}","index_url":"{release}"}}"#, + python.display() + ), + )?; + Ok(()) + })?; + outcome?; + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + // The torch install, the trio realign and the device realign are the real + // installs that name a torch-stack package; each must ask the SDK's index. + let torch_stack_installs: Vec<&String> = calls + .iter() + .filter(|call| !call.contains("--dry-run") && call.contains("--no-deps")) + .filter(|call| call.contains("torch")) + .collect(); + assert_eq!( + torch_stack_installs.len(), + 3, + "expected torch install, trio realign and device realign: {calls:?}" + ); + for call in torch_stack_installs { + assert!(call.contains(&format!("--index-url {release}")), "{call}"); + assert!(!call.contains(build.torch_index_url), "{call}"); + } + Ok(()) + } + + /// Plants a runtime manifest for the interpreter, with `extra` JSON fields. + #[cfg(unix)] + fn plant_manifest(paths: &AppPaths, python: &Path, extra: &str) -> Result<()> { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::write( + registry.join("runtime.json"), + format!(r#"{{"python_executable":"{}"{extra}}}"#, python.display()), + )?; + Ok(()) + } + /// Makes the registry unreadable: a directory where a manifest file belongs + /// cannot be read as a file by anyone, root included, unlike a mode-000 registry. + #[cfg(unix)] + fn make_registry_unreadable(paths: &AppPaths) -> Result<()> { + let broken = paths + .data_dir + .join("runtimes") + .join("registry") + .join("broken.json"); + std::fs::create_dir_all(broken)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn the_ten_two_row_says_when_it_falls_back_to_its_own_index() -> Result<()> { + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + let on_row_index = |calls: &[String]| { + calls + .iter() + .any(|call| call.contains(&format!("--index-url {}", build.torch_index_url))) + }; + + // A manifest that records no index (an older one, or an adopted runtime): + // the row's index, and the install says so. + let (outcome, calls) = + run_ten_two_discover_with_fake_uv(|paths, python| plant_manifest(paths, python, ""))?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("records a wheel index")), + "{notes:?}" + ); + assert!(on_row_index(&calls)); + + // A registry that cannot be read: the same index, and the install says so. + let (outcome, calls) = + run_ten_two_discover_with_fake_uv(|paths, _| make_registry_unreadable(paths))?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("could not read the runtime registry")), + "{notes:?}" + ); + assert!(on_row_index(&calls)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_row_that_discovers_its_stack_never_looks_at_the_registry_or_says_anything_about_it() + -> Result<()> { + // 10.1's index is fixed by design, so an unreadable registry or a manifest + // with no index is not something its install has a reason to mention. + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", None, |paths, _| { + make_registry_unreadable(paths) + })?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .all(|note| !note.contains("registry") && !note.contains("wheel index")), + "{notes:?}" + ); + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", None, |paths, python| { + plant_manifest(paths, python, "") + })?; + assert!( + outcome? + .notes + .iter() + .all(|note| !note.contains("wheel index")) + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_install_step_on_the_fallback_index_names_the_fallback() -> Result<()> { + // Each real install in turn: the torch install, the remaining deps, the trio + // realign and the device realign all run after the index fell back. + for step in 1..=4 { + let (outcome, _) = + run_discover_row_with_fake_uv("10.2.0", Some(step), |paths, python| { + plant_manifest(paths, python, "") + })?; + let err = format!("{:#}", outcome.expect_err("an install step fails")); + assert!(err.contains("boom"), "step {step}: {err}"); + assert!( + err.contains("records a wheel index") && err.contains("Fallbacks used"), + "step {step}: the uv failure must carry the note about the index it ran \ + against: {err}" + ); + } + + // With the SDK's own index recorded there is nothing to add to the failure. + let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", Some(1), |paths, python| { + plant_manifest( + paths, + python, + r#","index_url":"https://stable.example/whl""#, + ) + })?; + let err = format!("{:#}", outcome.expect_err("the torch install fails")); + assert!( + err.contains("boom") && !err.contains("Fallbacks used"), + "{err}" + ); + Ok(()) + } + #[cfg(unix)] #[test] fn discovery_installs_the_companions_the_discovered_vllm_depends_on() -> Result<()> { @@ -1317,6 +2571,102 @@ fi assert!(rendered.contains("10.1.0"), "{rendered}"); assert!(rendered.contains("rocm10.0.0"), "{rendered}"); } + /// The 10.2 vLLM wheels spell their ROCm line `.rocm102`, not + /// `+rocm10.2.0`; the guard must still compare it. + #[test] + fn rocm_local_version_guard_compares_the_compact_rocm_spelling() { + let ten_two = "vllm==0.32.1.dev0+g5df63a720.d20261110.rocm102"; + assert!(ensure_rocm_local_version_matches(ten_two, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(ten_two, "10.2.0a20261022").is_ok()); + let err = ensure_rocm_local_version_matches(ten_two, "10.1.0") + .expect_err("a 10.2 wheel must not install onto a 10.1 SDK"); + assert!(format!("{err:#}").contains("rocm102"), "{err:#}"); + // A tagged release carries the compact spelling as the whole local version. + let bare = "vllm==0.32.1+rocm102"; + assert!(ensure_rocm_local_version_matches(bare, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(bare, "10.1.0").is_err()); + // A `rocm` segment that is neither spelling names no line, so it is no match. + for unreadable in [ + "vllm==0.32.1+rocm", + "vllm==0.32.1+rocmabc", + "vllm==0.32.1+g5d.rocm.x", + ] { + assert!( + ensure_rocm_local_version_matches(unreadable, "10.2.0").is_err(), + "{unreadable}" + ); + } + // And the dotted spelling is read as a series wherever its segment sits. + let late = "vllm==0.32.1+g5df63a720.rocm10.2.0"; + assert!(ensure_rocm_local_version_matches(late, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(late, "10.3.0").is_err()); + // And a 10.1 wheel must not slip onto a 10.2 SDK through the dotted spelling. + assert!( + ensure_rocm_local_version_matches("vllm==0.29.1.dev0+rocm10.1.0.gabc", "10.2.0") + .is_err() + ); + } + /// A 10.2 runtime keeps the torch stack its SDK install resolved; a stack + /// from another series or another ROCm line is refused, not kept. + #[test] + fn installed_torch_stack_pins_reads_and_checks_the_sdk_stack() { + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); + let freeze = "flash-attn==2.8.3\ntorch==2.14.0+rocm10.2.0a20261022\n\ + torchaudio==2.11.1+rocm10.2.0a20261022\n\ + torchvision==0.29.2+rocm10.2.0a20261022\n"; + assert_eq!( + installed_torch_stack_pins(freeze, ten_two).expect("coherent 10.2 stack"), + [ + "torch==2.14.0+rocm10.2.0a20261022", + "torchvision==0.29.2+rocm10.2.0a20261022", + "torchaudio==2.11.1+rocm10.2.0a20261022", + ] + ); + // The 10.1 torch an unpinned SDK install can leave behind is not 2.14's pair. + let old = freeze.replace("2.14.0+rocm10.2.0a20261022", "2.13.0+rocm10.2.0a20261022"); + let err = installed_torch_stack_pins(&old, ten_two).expect_err("torch 2.13"); + assert!( + format!("{err:#}").contains("needs `torch` 2.14.x"), + "{err:#}" + ); + // Another ROCm line is refused by the same guard as every other pin. + let other_line = freeze.replace( + "torch==2.14.0+rocm10.2.0a20261022", + "torch==2.14.0+rocm10.3.0a20261110", + ); + assert!(installed_torch_stack_pins(&other_line, ten_two).is_err()); + // A runtime with no torch says what to do. + let err = installed_torch_stack_pins("flash-attn==2.8.3\n", ten_two).expect_err("no torch"); + assert!(format!("{err:#}").contains("rocm install sdk"), "{err:#}"); + } + /// An `Exact` torchaudio pin on a row that reuses the SDK's stack means that + /// release and no other (PEP 440 `==release`): the `+local` part is not + /// compared, and a longer release is not a match. + #[test] + fn installed_torch_stack_pins_treats_an_exact_pin_as_that_release() { + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); + let exact = |release: &'static str| VllmRocmDiscoverBuild { + torchaudio_version: DiscoverVersion::Exact(release), + ..*ten_two + }; + let freeze = |torchaudio: &str| { + format!( + "torch==2.14.0+rocm10.2.0a20261022\ntorchvision==0.29.2+rocm10.2.0a20261022\n\ + torchaudio=={torchaudio}+rocm10.2.0a20261022\n" + ) + }; + let accepted = installed_torch_stack_pins(&freeze("2.11.0.2"), &exact("2.11.0.2")) + .expect("the exact release with its local version"); + assert_eq!(accepted[2], "torchaudio==2.11.0.2+rocm10.2.0a20261022"); + for installed in ["2.11.0.2.9", "2.11.0.3", "2.11.1"] { + assert!( + installed_torch_stack_pins(&freeze(installed), &exact("2.11.0.2")).is_err(), + "{installed} is not the release 2.11.0.2" + ); + } + // A shorter pin does not accept the longer rebuild it is a prefix of. + assert!(installed_torch_stack_pins(&freeze("2.11.0.2"), &exact("2.11.0")).is_err()); + } /// Pins `install_vllm_rocm10_discover`'s own call sequencing, not just its /// arg-builder helpers in isolation: the remaining-deps install must use /// `--config-file` (never `--extra-index-url`, which cannot tolerate the diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 80ad31eca5..4fe7fc2d5f 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -18,6 +18,7 @@ use std::io::{Read, Write}; use std::path::PathBuf; mod discover_builds; +mod endpoint_auth; mod install; mod process; mod repair; @@ -190,8 +191,7 @@ pub fn run_cli() -> Result<()> { gpu_selection: crate::process::parse_gpu_selection_arg(gpu.as_deref())?, })?)?, CommandKind::Stdio => { - let envelope = read_request()?; - print_json(&handle_envelope(envelope))?; + print_json(&answer_stdio(&read_request()?))?; } CommandKind::ServeHttp { service_id, @@ -260,6 +260,11 @@ pub fn builtin_serve_http( } fn handle_envelope(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope { + // Refuse a request made for a protocol this engine cannot serve before its + // payload is read: in another protocol the payload may mean something else. + if let Some(refusal) = rocm_engine_protocol::incompatible_request_response(&envelope) { + return refusal; + } match envelope.method { EngineMethod::Detect => { deserialize_and_respond::(envelope.payload, |_| { @@ -309,7 +314,10 @@ where Ok(response) => EngineResponseEnvelope::success(response), Err(error) => EngineResponseEnvelope::failure("request_failed", error.to_string()), }, - Err(error) => EngineResponseEnvelope::failure("invalid_payload", error.to_string()), + Err(error) => EngineResponseEnvelope::failure( + rocm_engine_protocol::INVALID_PAYLOAD_CODE, + error.to_string(), + ), } } @@ -379,12 +387,21 @@ pub(crate) const fn windows_unsupported_message() -> &'static str { "vLLM ROCm serving is supported by rocm-cli only on Linux/WSL; native Windows vLLM is skipped. No CPU fallback is used." } -fn read_request() -> Result { +/// The request text on stdin, read to its end. It is decoded by +/// [`answer_stdio`], which judges its protocol version first. +fn read_request() -> Result { let mut buffer = String::new(); std::io::stdin() .read_to_string(&mut buffer) .context("failed to read stdin for engine request")?; - serde_json::from_str(&buffer).context("failed to parse engine request envelope") + Ok(buffer) +} + +/// The answer to the request `text` read on stdin: its protocol version is +/// judged before its method or payload is decoded (see +/// [`rocm_engine_protocol::answer_engine_request`]), then it is handled. +fn answer_stdio(text: &str) -> EngineResponseEnvelope { + rocm_engine_protocol::answer_engine_request(text, handle_envelope) } fn print_json(value: &T) -> Result<()> { @@ -457,36 +474,56 @@ mod tests { ]; for (method, payload) in success_cases { - let response = handle_envelope(EngineRequestEnvelope { method, payload }); + let response = handle_envelope(EngineRequestEnvelope::new(method, payload)); assert!( response.ok, "expected protocol method to return a typed success envelope: {:?}", response.error ); + assert_eq!( + response.protocol_version.as_deref(), + Some(rocm_engine_protocol::ENGINE_PROTOCOL_VERSION) + ); } - let endpoint = handle_envelope(EngineRequestEnvelope { - method: EngineMethod::Endpoint, - payload: json!({ + let endpoint = handle_envelope(EngineRequestEnvelope::new( + EngineMethod::Endpoint, + json!({ "service_id": service_id.as_str() }), - }); + )); assert!(!endpoint.ok); + assert_eq!( + endpoint.protocol_version.as_deref(), + Some(rocm_engine_protocol::ENGINE_PROTOCOL_VERSION) + ); assert_eq!( endpoint.error.as_ref().map(|error| error.code.as_str()), Some("request_failed") ); for method in [EngineMethod::Install, EngineMethod::Launch] { - let response = handle_envelope(EngineRequestEnvelope { - method, - payload: json!({}), - }); + let response = handle_envelope(EngineRequestEnvelope::new(method, json!({}))); assert!(!response.ok); + assert_eq!( + response.protocol_version.as_deref(), + Some(rocm_engine_protocol::ENGINE_PROTOCOL_VERSION) + ); assert_eq!( response.error.as_ref().map(|error| error.code.as_str()), - Some("invalid_payload") + Some(rocm_engine_protocol::INVALID_PAYLOAD_CODE) ); } } + + /// The engine protocol contract, the same check every engine runs: see + /// [`rocm_engine_protocol::engine_contract`]. + #[test] + fn stdio_answers_hold_the_engine_protocol_contract() { + rocm_engine_protocol::engine_contract::assert_engine_holds_protocol_contract( + "vllm", + answer_stdio, + handle_envelope, + ); + } } diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 87f3d4f9a5..2aa4c92ce7 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -127,6 +127,78 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { }) } +/// Tear down a server that never became ready: stop its process tree, reap it, +/// and record the service as `failed`. A state file that cannot be written is +/// reported on stderr rather than returned, so the readiness failure stays the +/// error the caller reports. +/// +/// A server still held unreaped is stopped with [`stop_spawned_server`] and +/// [`STOP_GRACE`]. When that stop is not confirmed, the reason is written to +/// stderr and the server is killed (`SIGKILL`): this process owns it and has not +/// reaped it, so its PID cannot belong to anyone else. That kill is what bounds +/// the wait after an `Unverified` stop, which signals nothing. The wait that +/// follows is therefore bounded on every path but one: a server that survives +/// `SIGKILL`, such as one in uninterruptible sleep in the kernel, is waited on +/// until it leaves that state. A process under a server whose stop was not +/// confirmed is not reached by the kill. +/// +/// The readiness poll may already have reaped the server, because it exited +/// before becoming ready. Its PID may then belong to an unrelated process, so +/// nothing is signalled, and any worker it left running is not reached either: +/// a reaped process can no longer be walked for its children. +fn fail_unready_server(child: &mut std::process::Child, state_path: &Path) { + fail_unready_server_with(child, state_path, |pid| { + stop_spawned_server(pid, STOP_GRACE) + }); +} + +/// [`fail_unready_server`] with the stop supplied by the caller, so a test can +/// drive an outcome a real process cannot be made to produce on demand. +fn fail_unready_server_with( + child: &mut std::process::Child, + state_path: &Path, + stop: impl FnOnce(u32) -> rocm_core::TerminationOutcome, +) { + let already_reaped = matches!(child.try_wait(), Ok(Some(_))); + if !already_reaped { + let outcome = stop(child.id()); + if let Some(reason) = outcome.unconfirmed_reason() { + eprintln!( + "Warning: process {} was not confirmed stopped: {reason}. Killing it; processes under it are not reached by that kill.", + child.id() + ); + let _ = child.kill(); + } + } + let _ = child.wait(); + if let Err(error) = crate::state::write_terminal_state(state_path, "failed") { + eprintln!( + "Warning: could not record the vLLM service as failed in {}: {error:#}", + state_path.display() + ); + } +} + +/// Force-stop the server tree this process spawned and still holds as an +/// unreaped child. +/// +/// `SIGTERM` to the whole tree, then `SIGKILL` for whatever outlives `grace`, so +/// the call returns within about twice `grace`. That alone does not bound a +/// following `child.wait()`: an `Unverified` outcome means nothing was signalled, +/// and a server that survives `SIGKILL` outlives the stop (`TimedOut`), so the +/// caller handles both ([`fail_unready_server_with`]). The identity is captured +/// here, which is safe only while the caller holds the unreaped child: until it +/// is reaped its PID cannot be reused. [`fail_unready_server_with`] checks +/// that before calling it. +fn stop_spawned_server(pid: u32, grace: Duration) -> rocm_core::TerminationOutcome { + rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(pid), + STOP_SCOPE, + grace, + true, + ) +} + pub(crate) fn serve_http(mut request: ServeHttpRequest) -> Result<()> { request.gpu_indices = resolve_serve_gpu_indices(&request.gpu_indices)?; let runtime = crate::runtime::resolve_vllm_runtime(request.runtime_id.as_deref())?; @@ -143,10 +215,12 @@ pub(crate) fn serve_http(mut request: ServeHttpRequest) -> Result<()> { request.log_path.as_deref(), ) { // Terminate the whole vLLM process tree so the EngineCore worker (which - // holds the GPU allocation) does not survive and leak device memory. - let _ = rocm_core::terminate_process_tree(child.id()); - let _ = child.wait(); - crate::state::write_terminal_state(&request.state_path, "failed")?; + // holds the GPU allocation) does not survive and leak device memory, and + // escalate so a server ignoring SIGTERM is killed rather than waited on. + // The teardown still blocks on a server that survives SIGKILL, such as + // one in uninterruptible sleep in the kernel. A server that already + // exited is not signalled; see `fail_unready_server`. + fail_unready_server(&mut child, &request.state_path); return Err(e); } @@ -235,25 +309,19 @@ the ROCm SDK's bundled numa uses renamed symbol versions and cannot satisfy it, .with_context(|| format!("failed to create {}", parent.display()))?; } - let mut command = ProcessCommand::new(&runtime.command); - command - .args(vllm_serve_args( - &request.model_ref, - &request.host, - request.port, - vllm_enforce_eager_enabled(), - request.engine_recipe.as_ref(), - )) - .stdin(Stdio::null()); - // When `rocm serve` protects a public endpoint, the key arrives via the - // environment / key file (never argv). Hand it to vLLM as `VLLM_API_KEY` so - // its OpenAI server rejects unauthenticated requests; passing it through the - // env keeps it out of the process table. - if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { - command.env("VLLM_API_KEY", api_key); - } - apply_therock_env(&mut command, runtime)?; - rocm_engine_protocol::apply_gpu_visibility(&mut command, &request.gpu_indices); + // When `rocm serve` protects an endpoint, the key arrives via the key file + // (never argv). It reaches vLLM through the environment, which keeps it out + // of the process table, and is enforced on every route by the middleware + // `endpoint_auth` installs: vLLM's own `VLLM_API_KEY` check alone leaves + // routes such as `/invocations` open. + let api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let mut command = vllm_server_command( + request, + runtime, + api_key.as_deref(), + || Ok(crate::endpoint_auth::module_dir(&AppPaths::discover()?)), + vllm_enforce_eager_enabled(), + )?; if let Some(log_path) = log_path { let log = fs::File::create(log_path) .with_context(|| format!("failed to create {}", log_path.display()))?; @@ -274,6 +342,40 @@ the ROCm SDK's bundled numa uses renamed symbol versions and cannot satisfy it, } }) } +/// The `vllm serve` command for `request`, short of where its output goes: +/// argv, TheRock environment and GPU visibility, plus, with an `api_key`, the +/// endpoint-key protection: the `--middleware` argument, `VLLM_API_KEY`, and +/// the middleware module written to `module_dir()` and put at the front of +/// `PYTHONPATH`. `--middleware` depends on the other two: vLLM has to import +/// the module from `PYTHONPATH`, and the middleware refuses to construct +/// without `VLLM_API_KEY`. So all three are built in one place and asserted +/// together by one test. `module_dir` is consulted only for a keyed launch. +fn vllm_server_command( + request: &ServeHttpRequest, + runtime: &VllmRuntime, + api_key: Option<&str>, + module_dir: impl FnOnce() -> Result, + enforce_eager: bool, +) -> Result { + let mut command = ProcessCommand::new(&runtime.command); + command + .args(vllm_serve_args( + &request.model_ref, + &request.host, + request.port, + enforce_eager, + request.engine_recipe.as_ref(), + api_key.is_some(), + )) + .stdin(Stdio::null()); + apply_therock_env(&mut command, runtime)?; + // After `apply_therock_env`, which may set its own `PYTHONPATH` entry. + if let Some(api_key) = api_key { + crate::endpoint_auth::protect_endpoint(&mut command, api_key, &module_dir()?)?; + } + rocm_engine_protocol::apply_gpu_visibility(&mut command, &request.gpu_indices); + Ok(command) +} /// Explain a spawn that failed on a script whose `#!` interpreter is gone. /// /// The kernel reports the missing *interpreter* as ENOENT against the *script*, @@ -516,7 +618,8 @@ fn apply_vllm_rocm10_discover_env( command.env("FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE"); Ok(()) } -/// Full argument vector for `vllm serve`, as spawned by [`spawn_vllm_server`]. +/// Full argument vector for `vllm serve`, as built into the launch command by +/// [`vllm_server_command`]. /// /// Kept as a pure function so the exact argv can be asserted in tests: memory /// and graph-mode flags materially change how much VRAM vLLM claims, and a @@ -526,12 +629,16 @@ fn apply_vllm_rocm10_discover_env( /// supply a default — vLLM's own default applies unless the user asks for a /// specific fraction via `rocm serve --gpu-memory-utilization`, which arrives /// here through the engine recipe's `required_flags`. +/// +/// `require_api_key` adds the `--middleware` that enforces the endpoint key on +/// every route (see [`crate::endpoint_auth`]). fn vllm_serve_args( model_ref: &str, host: &str, port: u16, enforce_eager: bool, engine_recipe: Option<&EngineRecipeHint>, + require_api_key: bool, ) -> Vec { let mut args = vec![ "serve".to_owned(), @@ -549,6 +656,9 @@ fn vllm_serve_args( args.push("--enforce-eager".to_owned()); } args.extend(engine_recipe_launch_args(engine_recipe)); + if require_api_key { + args.extend(crate::endpoint_auth::middleware_args()); + } args } /// Whether to launch vLLM with `--enforce-eager` (CUDA graphs disabled). @@ -583,7 +693,10 @@ fn resolve_vllm_ready_timeout(override_value: Option) -> Duration { .map_or(DEFAULT_VLLM_READY_TIMEOUT, Duration::from_secs) } -fn prepend_path_entries(entries: &[PathBuf], current: Option) -> Result { +pub(crate) fn prepend_path_entries( + entries: &[PathBuf], + current: Option, +) -> Result { let mut parts = Vec::new(); for entry in entries { if !entry.as_os_str().is_empty() && !parts.iter().any(|part: &PathBuf| part == entry) { @@ -815,7 +928,7 @@ mod tests { #[test] fn vllm_serve_args_omit_gpu_memory_utilization_without_override() { - let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None); + let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None, false); assert_eq!( args, @@ -842,7 +955,14 @@ mod tests { ..EngineRecipeHint::default() }; - let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, true, Some(&hint)); + let args = vllm_serve_args( + "facebook/opt-125m", + "127.0.0.1", + 8000, + true, + Some(&hint), + false, + ); let position = args .iter() @@ -852,6 +972,178 @@ mod tests { assert!(args.contains(&"--enforce-eager".to_owned())); } #[test] + fn vllm_serve_args_install_the_auth_middleware_when_a_key_is_required() { + let args = vllm_serve_args("facebook/opt-125m", "0.0.0.0", 8000, false, None, true); + + let position = args + .iter() + .position(|arg| arg == "--middleware") + .expect("a keyed endpoint must be launched with the auth middleware"); + assert_eq!( + args.get(position + 1).map(String::as_str), + Some(crate::endpoint_auth::MIDDLEWARE_IMPORT_PATH) + ); + assert!( + !args.iter().any(|arg| arg.contains("api-key")), + "the key itself must never reach argv: {args:?}" + ); + } + #[test] + fn vllm_serve_args_omit_the_auth_middleware_without_a_key() { + let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None, false); + assert!( + !args.iter().any(|arg| arg == "--middleware"), + "an unkeyed endpoint must not load the middleware, which refuses to construct \ + without a key: {args:?}" + ); + } + fn keyed_launch_request() -> ServeHttpRequest { + ServeHttpRequest { + service_id: "svc-vllm".to_owned(), + model_ref: "facebook/opt-125m".to_owned(), + host: "0.0.0.0".to_owned(), + port: 8000, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("state.json"), + log_path: None, + engine_recipe: None, + } + } + #[test] + fn keyed_vllm_server_command_installs_middleware_key_and_pythonpath_together() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + // A ROCm 10 discover runtime, whose TheRock environment sets its own + // `PYTHONPATH` entry (`amd_smi` in the venv). The middleware directory + // has to be prepended after that, or the TheRock value replaces it. + let venv = temp.path().join("venv"); + let site_packages = venv.join("lib").join("python3.12").join("site-packages"); + fs::create_dir_all(&site_packages)?; + let runtime = test_vllm_runtime( + Some(venv.join("bin").join("python")), + Some("10.0.0".to_owned()), + ); + let command = vllm_server_command( + &keyed_launch_request(), + &runtime, + Some("secret-key"), + || Ok(module_dir.clone()), + false, + )?; + + let args: Vec = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + let position = args + .iter() + .position(|arg| arg == "--middleware") + .expect("a keyed launch must pass the auth middleware to vllm serve"); + assert_eq!( + args.get(position + 1).map(String::as_str), + Some(crate::endpoint_auth::MIDDLEWARE_IMPORT_PATH) + ); + assert!( + !args.iter().any(|arg| arg.contains("secret-key")), + "the key must never reach argv: {args:?}" + ); + assert!( + !args.iter().any(|arg| arg == "--enforce-eager"), + "enforce_eager was false: {args:?}" + ); + assert_eq!( + crate::endpoint_auth::env_of(&command, "VLLM_API_KEY"), + Some("secret-key".into()), + "the middleware and vLLM's own check both read the key from VLLM_API_KEY" + ); + let pythonpath = crate::endpoint_auth::env_of(&command, "PYTHONPATH") + .expect("a keyed launch must put the middleware directory on PYTHONPATH"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + let amd_smi = site_packages + .join("_rocm_sdk_core") + .join("share") + .join("amd_smi"); + assert_eq!( + entries.get(..2), + Some(&[module_dir.clone(), amd_smi][..]), + "vLLM imports the middleware by module name, so its directory goes first, \ + ahead of the runtime's own entry, which is kept: {entries:?}" + ); + assert_eq!( + crate::endpoint_auth::env_of(&command, "HIP_VISIBLE_DEVICES"), + Some("1".into()), + "the GPU the request selected must still be applied to a keyed launch" + ); + assert!( + module_dir.join("rocm_cli_vllm_auth.py").is_file(), + "the middleware module must be written where PYTHONPATH points" + ); + Ok(()) + } + #[test] + fn vllm_server_command_forwards_enforce_eager() -> Result<()> { + let command = vllm_server_command( + &keyed_launch_request(), + &VllmRuntime::for_test(), + None, + || panic!("an unkeyed launch must not need the middleware directory"), + true, + )?; + assert!( + command.get_args().any(|arg| arg == "--enforce-eager"), + "enforce_eager was true: {:?}", + command.get_args().collect::>() + ); + Ok(()) + } + #[test] + fn unkeyed_vllm_server_command_carries_no_key_protection() -> Result<()> { + // The same TheRock runtime as the keyed test, so `PYTHONPATH` has a value + // of its own that a stray middleware entry would be prepended to. + let temp = tempfile::tempdir()?; + let venv = temp.path().join("venv"); + let site_packages = venv.join("lib").join("python3.12").join("site-packages"); + fs::create_dir_all(&site_packages)?; + let runtime = test_vllm_runtime( + Some(venv.join("bin").join("python")), + Some("10.0.0".to_owned()), + ); + let command = vllm_server_command( + &keyed_launch_request(), + &runtime, + None, + || panic!("an unkeyed launch must not need the middleware directory"), + false, + )?; + assert!( + !command.get_args().any(|arg| arg == "--middleware"), + "an unkeyed launch must not load the middleware, which refuses to start without a key" + ); + assert_eq!(crate::endpoint_auth::env_of(&command, "VLLM_API_KEY"), None); + let pythonpath = crate::endpoint_auth::env_of(&command, "PYTHONPATH") + .expect("the TheRock runtime sets its own PYTHONPATH entry"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + let amd_smi = site_packages + .join("_rocm_sdk_core") + .join("share") + .join("amd_smi"); + // The runtime's own value: its entry ahead of whatever PYTHONPATH this + // process inherited. + let expected: Vec = std::env::split_paths(&prepend_path_entries( + &[amd_smi], + std::env::var_os("PYTHONPATH"), + )?) + .collect(); + assert_eq!( + entries, expected, + "an unkeyed launch keeps the runtime's PYTHONPATH exactly, with no middleware directory" + ); + Ok(()) + } + #[test] fn resolve_model_echoes_matching_engine_recipe() -> Result<()> { let hint = test_engine_recipe(ENGINE_NAME, ENGINE_RECIPE_CONTRACT_VERSION); let response = resolve_model_response(ResolveModelRequest { @@ -1205,4 +1497,298 @@ mod tests { assert_eq!(flag, Some(std::ffi::OsStr::new("TRUE"))); Ok(()) } + + /// Spawn `sh -c script`, which prints one line once it is ready, and return + /// the child with that line. + #[cfg(target_os = "linux")] + fn spawn_ready(script: &str) -> (std::process::Child, String) { + use std::io::BufRead; + let mut child = ProcessCommand::new("sh") + .args(["-c", script]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn test child"); + let mut line = String::new(); + std::io::BufReader::new(child.stdout.take().expect("piped stdout")) + .read_line(&mut line) + .expect("read readiness line"); + (child, line.trim().to_owned()) + } + + /// `SIGKILL` `identity` if it is still that process, so a failing test does + /// not leave a stray process behind. + #[cfg(target_os = "linux")] + fn kill_if_running(identity: &rocm_core::ProcessIdentity) { + if rocm_core::identity_state(identity) == rocm_core::IdentityState::Matches { + let _ = ProcessCommand::new("kill") + .args(["-KILL", &identity.pid.to_string()]) + .status(); + } + } + + /// The `sh` script for a server that ignores `SIGTERM` and has a worker that + /// ignores it too (`SIG_IGN` survives `exec`). The inner shell prints its + /// PID once it runs, which is after the outer trap is set. + #[cfg(target_os = "linux")] + const TRAPPING_TREE: &str = r#"trap '' TERM; sh -c "echo \$\$; exec sleep 300" & wait"#; + + /// Run `teardown` on a fresh [`TRAPPING_TREE`] in a thread, bounded so a + /// teardown that hangs fails the test instead, and return the reaped server's + /// exit status, whether the worker is gone, and how long the teardown took. + #[cfg(target_os = "linux")] + fn tear_down_trapping_tree( + bound: Duration, + teardown: impl FnOnce(&mut std::process::Child) + Send + 'static, + ) -> (std::process::ExitStatus, bool, Duration) { + let (mut child, worker_pid) = spawn_ready(TRAPPING_TREE); + let worker = rocm_core::ProcessIdentity::capture(worker_pid.parse().expect("worker pid")); + let server = rocm_core::ProcessIdentity::capture(child.id()); + let (done, finished) = std::sync::mpsc::channel(); + let thread = std::thread::spawn(move || { + let started = Instant::now(); + teardown(&mut child); + let _ = done.send(started.elapsed()); + child + }); + let elapsed = finished.recv_timeout(bound); + if elapsed.is_err() { + // Unblock the teardown's `wait` so the test fails rather than hangs. + kill_if_running(&worker); + kill_if_running(&server); + } + let mut child = thread.join().expect("teardown thread"); + let worker_gone = matches!( + rocm_core::identity_state(&worker), + rocm_core::IdentityState::Gone | rocm_core::IdentityState::Recycled + ); + kill_if_running(&worker); + let elapsed = + elapsed.unwrap_or_else(|_| panic!("the teardown did not return within {bound:?}")); + let status = child + .try_wait() + .expect("try_wait") + .expect("the teardown reaps the server"); + (status, worker_gone, elapsed) + } + + /// The readiness-failure teardown with a real stop and a short grace: a + /// server that ignores `SIGTERM`, and its worker, are killed, the teardown + /// returns within about twice the grace period, and the service is recorded + /// as failed. + #[cfg(target_os = "linux")] + #[test] + fn fail_unready_server_kills_a_tree_that_ignores_sigterm_and_records_failed() { + use std::os::unix::process::ExitStatusExt; + let grace = Duration::from_millis(500); + let dir = tempfile::tempdir().expect("tempdir"); + let state_path = dir.path().join("state.json"); + let thread_state_path = state_path.clone(); + + let (status, worker_gone, _) = + // The margin is wide: the bound only turns a hang into a failure, and a + // loaded host can be slow to run the stop. + tear_down_trapping_tree(grace * 2 + Duration::from_secs(10), move |child| { + fail_unready_server_with(child, &thread_state_path, |pid| { + stop_spawned_server(pid, grace) + }); + }); + + assert_eq!(status.signal(), Some(9), "{status}"); + assert!( + worker_gone, + "the worker under the server must be stopped too" + ); + let state = crate::state::read_service_state(&state_path).expect("state"); + assert_eq!(state["status"], "failed"); + } + + /// `fail_unready_server` itself, as `serve_http` calls it: the production + /// grace passes before the escalation, then the tree is killed and the + /// service recorded as failed. Takes about [`STOP_GRACE`]. + #[cfg(target_os = "linux")] + #[test] + fn fail_unready_server_escalates_after_the_stop_grace() { + use std::os::unix::process::ExitStatusExt; + let dir = tempfile::tempdir().expect("tempdir"); + let state_path = dir.path().join("state.json"); + let thread_state_path = state_path.clone(); + + let (status, worker_gone, elapsed) = + tear_down_trapping_tree(STOP_GRACE * 2 + Duration::from_secs(5), move |child| { + fail_unready_server(child, &thread_state_path); + }); + + assert!( + elapsed >= STOP_GRACE, + "the server must get the whole stop grace before SIGKILL: {elapsed:?}" + ); + assert_eq!(status.signal(), Some(9), "{status}"); + assert!( + worker_gone, + "the worker under the server must be stopped too" + ); + let state = crate::state::read_service_state(&state_path).expect("state"); + assert_eq!(state["status"], "failed"); + } + + /// A server the readiness poll already reaped is not signalled: its PID may + /// belong to another process by now. + #[cfg(target_os = "linux")] + #[test] + fn fail_unready_server_does_not_signal_a_server_the_readiness_poll_reaped() { + let mut child = ProcessCommand::new("true") + .stdin(Stdio::null()) + .spawn() + .expect("spawn test child"); + let deadline = Instant::now() + Duration::from_secs(5); + while child.try_wait().expect("try_wait").is_none() { + assert!(Instant::now() < deadline, "the child did not exit"); + std::thread::sleep(Duration::from_millis(10)); + } + let dir = tempfile::tempdir().expect("tempdir"); + let state_path = dir.path().join("state.json"); + let mut signalled = Vec::new(); + + fail_unready_server_with(&mut child, &state_path, |pid| { + signalled.push(pid); + rocm_core::TerminationOutcome::Forced + }); + + assert!( + signalled.is_empty(), + "a reaped server's PID must not be signalled: {signalled:?}" + ); + let state = crate::state::read_service_state(&state_path).expect("state"); + assert_eq!(state["status"], "failed"); + } + + /// What `fail_unready_server` writes to stderr, read from the real stderr of + /// a process running it, together with the state it describes. The test + /// re-runs itself in a child process (selected by an environment variable), + /// because a warning on this process's stderr cannot be read back here. + /// + /// - A stop that sends nothing and reports `Unverified` is warned about, and + /// the server is then killed and reaped rather than waited on. + /// - A state file that cannot be written is warned about, and the teardown + /// still completes. + /// - A confirmed stop is not warned about and is not followed by a kill, and + /// the server is reaped before the teardown returns. + #[cfg(target_os = "linux")] + #[test] + fn fail_unready_server_reports_on_stderr_and_kills_a_server_it_could_not_confirm_stopped() { + use std::os::unix::process::ExitStatusExt; + const ARM: &str = "ROCM_TEST_VLLM_UNREADY_TEARDOWN_ARM"; + if let Some(dir) = std::env::var_os(ARM) { + let dir = std::path::PathBuf::from(dir); + // Bounded, so a teardown that fails to kill it ends within 30 s + // instead of hanging the parent. + let (mut child, _) = spawn_ready("echo ready; exec sleep 30"); + let pid = child.id(); + let state_path = dir.join("state.json"); + fail_unready_server_with(&mut child, &state_path, |_| { + rocm_core::TerminationOutcome::Unverified + }); + let signal = child + .try_wait() + .expect("try_wait") + .and_then(|status| status.signal()); + // On lines of their own: the harness has already printed the test's + // name without a line break. + println!("\nunverified={pid} signal={signal:?}"); + + // A state path that is a directory cannot be written as a file. + let (mut child, _) = spawn_ready("echo ready; exec sleep 30"); + let unwritable = dir.join("unwritable"); + std::fs::create_dir(&unwritable).expect("create the unwritable path"); + fail_unready_server_with(&mut child, &unwritable, |pid| { + stop_spawned_server(pid, Duration::from_millis(500)) + }); + println!( + "\nunwritable reaped={}", + child.try_wait().expect("try_wait").is_some() + ); + + // A confirmed stop that leaves the server to exit by itself, with a + // status only that exit produces: a kill would end it with signal 9, + // and a teardown that did not wait would return while it still runs. + let (mut child, _) = spawn_ready("echo ready; sleep 1; exit 7"); + let pid = child.id(); + fail_unready_server_with(&mut child, &dir.join("confirmed.json"), |_| { + rocm_core::TerminationOutcome::Graceful + }); + let status = child + .try_wait() + .expect("try_wait") + .map(|status| (status.code(), status.signal())); + println!("\nconfirmed={pid} status={status:?}"); + return; + } + + let dir = tempfile::tempdir().expect("tempdir"); + let output = ProcessCommand::new(std::env::current_exe().expect("current test executable")) + .args([ + "process::tests::fail_unready_server_reports_on_stderr_and_kills_a_server_it_could_not_confirm_stopped", + "--exact", + "--nocapture", + ]) + .env(ARM, dir.path()) + .stdin(Stdio::null()) + .output() + .expect("re-run the test arm in a child process"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + let pid: u32 = stdout + .lines() + .find_map(|line| line.strip_prefix("unverified=")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|pid| pid.parse().ok()) + .unwrap_or_else(|| panic!("no unverified= line in: {stdout}")); + + assert!( + stdout.contains(&format!("unverified={pid} signal=Some(9)")), + "a server whose stop was not confirmed is killed: {stdout}" + ); + assert!( + stderr.contains(&format!( + "Warning: process {pid} was not confirmed stopped: its identity could not be \ + confirmed, so it was not signalled. Killing it; processes under it are not \ + reached by that kill." + )), + "{stderr}" + ); + let state = + crate::state::read_service_state(&dir.path().join("state.json")).expect("state"); + assert_eq!(state["status"], "failed"); + + let confirmed: u32 = stdout + .lines() + .find_map(|line| line.strip_prefix("confirmed=")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|pid| pid.parse().ok()) + .unwrap_or_else(|| panic!("no confirmed= line in: {stdout}")); + assert!( + stdout.contains(&format!( + "confirmed={confirmed} status=Some((Some(7), None))" + )), + "a confirmed stop is neither followed by a kill nor left unreaped: {stdout}" + ); + assert!( + !stderr.contains(&format!("process {confirmed} was not confirmed stopped")), + "a confirmed stop must not warn: {stderr}" + ); + let state = + crate::state::read_service_state(&dir.path().join("confirmed.json")).expect("state"); + assert_eq!(state["status"], "failed"); + + assert!(stdout.contains("unwritable reaped=true"), "{stdout}"); + assert!( + stderr.contains(&format!( + "Warning: could not record the vLLM service as failed in {}", + dir.path().join("unwritable").display() + )), + "{stderr}" + ); + } } diff --git a/engines/vllm/src/repair.rs b/engines/vllm/src/repair.rs index 9718abe3e7..d4907086f6 100644 --- a/engines/vllm/src/repair.rs +++ b/engines/vllm/src/repair.rs @@ -61,6 +61,17 @@ fn torch_alignment_disabled() -> bool { rocm_core::torch_alignment_disabled() } +/// Whether this violation is the tensorizer upgrade the install makes on purpose. +/// +/// vLLM pins `tensorizer==2.10.1`, which cannot be imported under Python 3.14; the +/// install upgrades it (see `ensure_runtime_tensorizer` in `install.rs`), so +/// `uv pip check` reports the exact pin as unmet forever. Treating that as a defect +/// would reinstall vLLM, and with it the broken tensorizer, on every +/// `rocm engines install vllm`. +fn is_intended_tensorizer_override(detail: &str) -> bool { + rocm_core::vllm_tensorizer_override_installed(detail).is_some() +} + /// Whether this violation is the torch divergence rocm-cli deliberately leaves behind. /// /// After an engine install, rocm-cli puts back the SDK's *build* of the torch release @@ -153,27 +164,44 @@ fn repair_from_violations( } let (intended, defects): (Vec<&DependencyViolation>, Vec<&DependencyViolation>) = owned.into_iter().partition(|violation| { - is_intended_torch_divergence( - &violation.detail, - sdk_torch_build, - torch_alignment_disabled, - ) + is_intended_tensorizer_override(&violation.detail) + || is_intended_torch_divergence( + &violation.detail, + sdk_torch_build, + torch_alignment_disabled, + ) }); if defects.is_empty() { - // Nothing to repair. Reinstalling here would replace that torch with the - // engine's build and hand back a runtime nobody asked for. + // Nothing to repair: every violation is an intended divergence. For torch, + // reinstalling would replace the SDK's build with the engine's and hand back a + // runtime nobody asked for. For tensorizer the install keeps the upgrade, so + // a reinstall is not needed (it would only upgrade tensorizer again). // // Which sentence is true depends on whose torch this is. Under the alignment it // is the SDK's and rocm-cli put it there; under the opt-out it is the user's and // rocm-cli never touched it. Reusing the first line for the second case would // tell a user who hand-installed torch that the CLI had installed it for them. - let headline = if torch_alignment_disabled { + let has_tensorizer = intended + .iter() + .any(|violation| is_intended_tensorizer_override(&violation.detail)); + let has_torch = intended + .iter() + .any(|violation| !is_intended_tensorizer_override(&violation.detail)); + let torch_headline = if torch_alignment_disabled { "torch alignment is disabled by ROCM_CLI_DISABLE_TORCH_ALIGNMENT; the torch stack this runtime holds is the user's and a reinstall would replace it" } else { "the runtime holds the SDK's build of the torch vLLM pins; that divergence is intended and a reinstall would undo it" }; - let mut notes = vec![headline.to_owned()]; + let tensorizer_headline = "the runtime holds a tensorizer at or above the fixed release instead of vLLM's older pin; that divergence is intended and the install keeps it, so no reinstall is needed"; + // One note per kind of divergence, not one joined line. + let mut notes = Vec::new(); + if has_torch || !has_tensorizer { + notes.push(torch_headline.to_owned()); + } + if has_tensorizer { + notes.push(tensorizer_headline.to_owned()); + } notes.extend( intended .iter() @@ -250,6 +278,124 @@ mod tests { const ALIGNED: bool = false; const OPTED_OUT: bool = true; + const TENSORIZER_UNMET: &str = + "The package `vllm` requires `tensorizer==2.10.1`, but `2.12.1` is installed"; + + #[test] + fn the_tensorizer_override_is_not_a_reason_to_reinstall() { + // The install upgrades vLLM's tensorizer pin on purpose because the pinned + // release cannot be imported on some Pythons. The install keeps the upgrade, so + // no reinstall is needed. + let assessment = repair_from_violations( + &[violation("vllm", TENSORIZER_UNMET)], + Some(SDK_BUILD), + ALIGNED, + ); + assert!(!assessment.needed, "{assessment:?}"); + // The headline, not the `expected divergence:` line below it, is what says + // why nothing is reinstalled, and it is about tensorizer alone. + let headline = &assessment.notes[0]; + assert!( + headline.contains("tensorizer") && headline.contains("no reinstall is needed"), + "{headline}" + ); + assert!(!headline.contains("torch"), "{headline}"); + assert!(headline.len() < 200, "{headline}"); + } + + #[test] + fn a_torch_and_a_tensorizer_divergence_together_are_both_named_in_the_headline() { + let torch = "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed"; + let assessment = repair_from_violations( + &[ + violation("vllm", torch), + violation("vllm", TENSORIZER_UNMET), + ], + Some(SDK_BUILD), + ALIGNED, + ); + assert!(!assessment.needed, "{assessment:?}"); + assert!( + assessment.notes[0].contains("SDK's build of the torch") + && !assessment.notes[0].contains("tensorizer"), + "{:?}", + assessment.notes + ); + assert!( + assessment.notes[1].contains("tensorizer") + && assessment.notes[1].contains("no reinstall is needed") + && !assessment.notes[1].contains("torch"), + "{:?}", + assessment.notes + ); + // The two headlines are each under 200 characters, not one joined line. The + // `expected divergence:` lines after them carry the raw uv text and are not + // bounded by this. + assert!( + assessment.notes[..2].iter().all(|note| note.len() < 200), + "the headlines should not be a wall of joined findings: {:?}", + assessment.notes + ); + } + + #[test] + fn a_marker_qualified_or_unreadable_pin_still_forces_a_reinstall() { + // `violation_subject` reads the text after `==` as the required release, so + // a pin qualified by an environment marker is not a plain release below the + // fix and must not pass as the deliberate override. + for pin in [ + "2.10.1; python_version >= \"3.14\"", + "garbage", + "2.10.1.postgarbage", + ] { + let detail = format!( + "The package `vllm` requires `tensorizer=={pin}`, but `2.12.1` is installed" + ); + let subject = violation_subject(&detail).expect("a requirement line parses"); + assert_eq!(subject.required.as_deref(), Some(pin), "{detail}"); + let assessment = + repair_from_violations(&[violation("vllm", &detail)], Some(SDK_BUILD), ALIGNED); + assert!(assessment.needed, "{pin:?}: {assessment:?}"); + } + } + + #[test] + fn a_tensorizer_older_than_the_fix_or_missing_still_forces_a_reinstall() { + for detail in [ + "The package `vllm` requires `tensorizer==2.13.0`, but `2.12.1` is installed", + "The package `vllm` requires `tensorizer==2.10.1`, but `2.9.0` is installed", + "The package `vllm` requires `tensorizer==2.10.1`, but it's not installed", + ] { + let assessment = + repair_from_violations(&[violation("vllm", detail)], Some(SDK_BUILD), ALIGNED); + assert!(assessment.needed, "{detail}: {assessment:?}"); + } + } + + #[test] + fn the_tensorizer_override_does_not_hide_a_genuine_violation() { + let assessment = repair_from_violations( + &[ + violation("vllm", TENSORIZER_UNMET), + violation( + "vllm", + "The package `vllm` requires `numpy==2.0.0`, but `1.26.0` is installed", + ), + ], + Some(SDK_BUILD), + ALIGNED, + ); + assert!(assessment.needed); + assert!( + assessment + .notes + .iter() + .any(|note| note.starts_with("expected divergence:") && note.contains("tensorizer")), + "{:?}", + assessment.notes + ); + } + #[test] fn a_consistent_environment_is_not_reinstalled() { // The other settled state: the SDK published no build of the release vLLM diff --git a/engines/vllm/src/rocm_cli_vllm_auth.py b/engines/vllm/src/rocm_cli_vllm_auth.py new file mode 100644 index 0000000000..38f408f811 --- /dev/null +++ b/engines/vllm/src/rocm_cli_vllm_auth.py @@ -0,0 +1,125 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +"""Require the endpoint API key on every route of a vLLM OpenAI server. + +rocm-cli passes ``--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`` to +``vllm serve`` whenever it protects an endpoint with an API key. vLLM's own +``VLLM_API_KEY`` check guards only a fixed set of path prefixes (``/v1`` and +a few others), while the server also registers inference-capable routes +outside them (``/invocations``, ``/tokenize``, ...). This middleware closes +that gap by checking the key on every HTTP and WebSocket request, whatever +its path, so a route vLLM adds in a later release is covered too. + +Only ``GET``/``HEAD /health`` and CORS preflight requests pass without the +key: the first is the liveness probe rocm-cli advertises, the second carries +no credentials by design and is answered by vLLM's CORS middleware before +any route runs. + +The key is read from ``VLLM_API_KEY`` -- the same variable vLLM's own check +reads, set by rocm-cli on the server process -- so both checks always agree. +Construction fails when the variable is empty, so a launch that asked for +protection cannot come up unprotected. + +Pure ASGI and standard library only: it must import in whichever vLLM +release the runtime ships, with no assumption about its Starlette version. +This file is written out by the rocm-cli vLLM engine adapter at launch; +edit the copy in the rocm-cli source tree, not the one on disk. +""" + +import hashlib +import hmac +import os + +KEY_ENV = "VLLM_API_KEY" + +# Exact paths (after any ASGI root_path) that answer without the key, and the +# only methods they answer to. Everything else requires the key. +UNAUTHENTICATED_ROUTES = {"/health": frozenset({"GET", "HEAD"})} + +_UNAUTHORIZED_BODY = b'{"error":"Unauthorized"}' + +# RFC 6455 policy-violation close code. Closing before accepting makes the +# server reject the WebSocket handshake with HTTP 403. +_WS_POLICY_VIOLATION = 1008 + + +def _digest(value): + return hashlib.sha256(value).digest() + + +class RequireEndpointApiKey: + """ASGI middleware rejecting any request that lacks the bearer key.""" + + def __init__(self, app): + key = os.environ.get(KEY_ENV, "") + if not key: + raise RuntimeError( + f"{KEY_ENV} is empty; refusing to start an endpoint that was " + "asked to require an API key without one" + ) + self.app = app + self._expected = _digest(key.encode("utf-8")) + + def _authorized(self, scope): + authorization = None + for name, value in scope.get("headers") or (): + if name.lower() == b"authorization": + authorization = value + break + if authorization is None: + return False + scheme, _, token = authorization.decode("latin-1").partition(" ") + if scheme.lower() != "bearer": + return False + # Compare fixed-length digests so the comparison time depends on + # neither the key's length nor how much of it matched. + return hmac.compare_digest(_digest(token.encode("utf-8")), self._expected) + + @staticmethod + def _route_path(scope): + path = scope.get("path", "") + root_path = scope.get("root_path", "") + if root_path and path.startswith(root_path): + path = path[len(root_path) :] + return path + + @staticmethod + def _is_cors_preflight(scope): + if scope.get("method") != "OPTIONS": + return False + names = {name.lower() for name, _ in scope.get("headers") or ()} + return b"origin" in names and b"access-control-request-method" in names + + def _exempt(self, scope): + if scope["type"] != "http": + return False + if self._is_cors_preflight(scope): + return True + methods = UNAUTHENTICATED_ROUTES.get(self._route_path(scope)) + return methods is not None and scope.get("method") in methods + + async def __call__(self, scope, receive, send): + kind = scope["type"] + if kind not in ("http", "websocket") or self._exempt(scope): + await self.app(scope, receive, send) + return + if self._authorized(scope): + await self.app(scope, receive, send) + return + if kind == "websocket": + await send({"type": "websocket.close", "code": _WS_POLICY_VIOLATION}) + return + await send( + { + "type": "http.response.start", + "status": 401, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(_UNAUTHORIZED_BODY)).encode()), + (b"www-authenticate", b"Bearer"), + ], + } + ) + await send({"type": "http.response.body", "body": _UNAUTHORIZED_BODY}) diff --git a/engines/vllm/src/runtime.rs b/engines/vllm/src/runtime.rs index 29b87b2e39..228ce53da0 100644 --- a/engines/vllm/src/runtime.rs +++ b/engines/vllm/src/runtime.rs @@ -78,6 +78,10 @@ pub(crate) struct TheRockRuntimeManifest { pub sdk_torch: Option, #[serde(default)] pub installed_at_unix_ms: Option, + /// The wheel index the SDK install resolved its packages from (nightly or + /// release); absent in manifests written before it was recorded. + #[serde(default)] + pub index_url: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -356,6 +360,29 @@ fn sort_manifests_newest_install_first(manifests: &mut [(PathBuf, TheRockRuntime .then_with(|| left_path.cmp(right_path)) }); } +/// The wheel index the SDK in the runtime whose interpreter is `python` was installed +/// from, as its manifest in `paths`' runtime registry records it. `Ok(None)` means no +/// manifest for that interpreter recorded one; `Err` means the registry could not be read. +/// +/// A discover row that reuses the SDK's own torch stack has to fetch those exact builds +/// again from the index they came from: a nightly SDK's torch is not on the release +/// index and a release SDK's is not on the nightly one. +pub(crate) fn recorded_sdk_index_url(paths: &AppPaths, python: &Path) -> Result> { + let registry = paths.data_dir.join("runtimes").join("registry"); + let manifests = load_runtime_manifests_in(®istry, None)?; + Ok(sdk_index_url_for_python(&manifests, python)) +} +/// The `index_url` of the manifest whose interpreter is `python`, if it recorded a +/// non-empty one. +fn sdk_index_url_for_python(manifests: &[TheRockRuntimeManifest], python: &Path) -> Option { + manifests + .iter() + .find(|manifest| manifest.python_executable.as_deref() == Some(python)) + .and_then(|manifest| manifest.index_url.as_deref()) + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(str::to_owned) +} /// The torch build the SDK installed into this runtime, as its manifest records it. /// /// Read from the manifest, never from the environment. By the time this runs the @@ -751,8 +778,9 @@ fn compat_dir_has_shims(compat_dir: &Path) -> bool { /// from the launch path, before building the environment a server is spawned /// with — never from running-state payload assembly, which only needs the /// path list `therock_library_path_entries` derives, not the shims -/// themselves. Returns the `libmpi_cxx` compat dir when a shim is needed for -/// *this* launch and `ensure_mpi_cxx_compat` wrote it — pass this on to +/// themselves. Returns the compat dir when a shim is needed for *this* launch: +/// the `libmpi_cxx` stub `ensure_mpi_cxx_compat` wrote, or the amd-smi link +/// [`rocm_core::ensure_amd_smi_compat`] wrote. Pass this on to /// [`therock_library_path_entries_for_launch`] rather than having that /// derivation re-check the directory itself. Returns `None` without touching /// the filesystem when `runtime.sdk_root` is absent, so this function stays @@ -782,6 +810,20 @@ pub(crate) fn ensure_therock_compat_shims( // binding symbols torch needs; they are never called in single-node // serving, so this is safe. let mpi_cxx_shim = ensure_mpi_cxx_compat(&compat_dir); + // The SDK's `amdsmi` package opens `libamd_smi.so` by its bare name, which the + // SDK does not ship, so it loads a second amd-smi library beside torch's own + // and torch counts no devices (see `rocm_core::ensure_amd_smi_compat`). The + // link goes in the same directory, and the launch path carries the directory + // when either shim was written. That function also removes a leftover + // `libmpi_cxx.so.40` stub where the host has a real `libmpi_cxx`, and a stale + // `libnuma.so.1` link, since the directory can now reach the loader path for + // the link alone. When it answers `None` the link is not written, but the + // directory still reaches this launch's loader path if only the mpi stub was + // written, which is why the numa link is also cleared directly below. + let amd_smi_shim = rocm_core::ensure_amd_smi_compat( + &compat_dir, + &therock_library_path_entries_inner(runtime, CompatDirSignal::Checked(None)), + ); // PyTorch's `libc10.so` NEEDS the standard `libnuma.so.1` soname with the // upstream `libnuma_1.2` symbol version. TheRock bundles numa only under // the renamed soname `librocm_sysdeps_numa.so.1` whose versions are @@ -792,29 +834,16 @@ pub(crate) fn ensure_therock_compat_shims( // that stale shim here so the real numactl runtime (installed via the // package manager) wins; `libnuma_present()`/`spawn_vllm_server` handle // the install/preflight. - remove_stale_numa_shim(&compat_dir); - mpi_cxx_shim -} -/// Remove a stale `libnuma.so.1` compatibility symlink left by older rocm-cli -/// versions in `compat_dir`. That shim pointed at the ROCm SDK's bundled -/// `librocm_sysdeps_numa.so.1`, whose renamed symbol versions cannot satisfy the -/// `libnuma_1.2` symbol PyTorch's `libc10.so` binds; leaving it on the loader -/// path would shadow a correctly installed system libnuma. No-op when absent or -/// when the entry is not a symlink. -fn remove_stale_numa_shim(compat_dir: &Path) { - let link = compat_dir.join("libnuma.so.1"); - if let Ok(meta) = link.symlink_metadata() - && meta.file_type().is_symlink() - { - let _ = fs::remove_file(&link); - } + rocm_core::remove_stale_numa_shim(&compat_dir); + mpi_cxx_shim.or(amd_smi_shim) } + /// A writable, runtime-owned directory for managed-runtime library compatibility /// shims (see [`therock_library_path_entries`]). Prefers the managed Python /// environment root (`/bin/python` -> ``); falls back to the SDK root /// when no Python launcher is recorded. fn runtime_compat_dir(runtime: &VllmRuntime) -> Option { - const COMPAT_DIR_NAME: &str = "rocm-cli-lib-compat"; + const COMPAT_DIR_NAME: &str = rocm_core::RUNTIME_COMPAT_DIR_NAME; if let Some(env_root) = runtime .python_executable .as_ref() @@ -843,6 +872,38 @@ fn dedupe_paths(entries: Vec) -> Vec { mod tests { use super::*; + #[test] + fn the_sdk_index_comes_from_the_manifest_of_that_interpreter() { + let manifest = |python: &str, index: Option<&str>| TheRockRuntimeManifest { + python_executable: Some(PathBuf::from(python)), + index_url: index.map(str::to_owned), + ..Default::default() + }; + let manifests = vec![ + manifest( + "/rt/nightly/bin/python", + Some("https://nightly.example/whl"), + ), + manifest( + "/rt/release/bin/python", + Some(" https://stable.example/whl "), + ), + manifest("/rt/old/bin/python", None), + manifest("/rt/blank/bin/python", Some(" ")), + ]; + let lookup = |python: &str| sdk_index_url_for_python(&manifests, Path::new(python)); + assert_eq!( + lookup("/rt/nightly/bin/python").as_deref(), + Some("https://nightly.example/whl") + ); + assert_eq!( + lookup("/rt/release/bin/python").as_deref(), + Some("https://stable.example/whl") + ); + assert_eq!(lookup("/rt/old/bin/python"), None); + assert_eq!(lookup("/rt/blank/bin/python"), None); + assert_eq!(lookup("/rt/unknown/bin/python"), None); + } #[test] fn managed_env_reflects_managed_runtime_manifest_source() { let mut runtime = VllmRuntime { @@ -986,11 +1047,14 @@ mod tests { assert!(runtime_has_discover_build(&runtime_with_sdk_version(Some( "10.1.0a20260611" )))); + assert!(runtime_has_discover_build(&runtime_with_sdk_version(Some( + "10.2.0" + )))); } #[test] fn runtime_has_discover_build_rejects_a_line_with_no_discover_row() { assert!(!runtime_has_discover_build(&runtime_with_sdk_version( - Some("10.2.0") + Some("10.3.0") ))); assert!(!runtime_has_discover_build(&runtime_with_sdk_version( Some("7.2.3") @@ -1159,6 +1223,132 @@ mod tests { } } + /// The SDK ships `libamd_smi.so.` only, and the `amdsmi` package opens the + /// bare name, so a launch needs the link on its loader path or torch counts no + /// devices beside a host ROCm. The mpi shim answers `None` here, as it does on + /// a host with a real `libmpi_cxx`: the amd-smi link alone must reach the path. + #[cfg(target_os = "linux")] + #[test] + fn a_launch_carries_the_amd_smi_link_even_when_no_mpi_shim_is_due() { + let env = tempfile::tempdir().unwrap(); + let sdk_root = env.path().join("sdk"); + let sdk_lib = sdk_root.join("lib"); + fs::create_dir_all(&sdk_lib).unwrap(); + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk").unwrap(); + let runtime = VllmRuntime { + runtime_id: "release-wheel".to_owned(), + env_id: "release-wheel".to_owned(), + python_executable: Some(env.path().join("bin").join("python")), + sdk_root: Some(sdk_root), + sdk_library_paths: vec![sdk_lib.clone()], + ..VllmRuntime::for_test() + }; + + let shim = ensure_therock_compat_shims(&runtime, |_| None); + + let compat_dir = env.path().join("rocm-cli-lib-compat"); + assert_eq!(shim.as_deref(), Some(compat_dir.as_path())); + assert_eq!( + fs::read_link(compat_dir.join("libamd_smi.so")).unwrap(), + sdk_lib.join("libamd_smi.so.26") + ); + let entries = therock_library_path_entries_for_launch(&runtime, shim); + assert!( + entries.contains(&compat_dir), + "the link only helps when its directory is on LD_LIBRARY_PATH: {entries:?}" + ); + // After the SDK's own directory, so a real bare name found there wins. + let position = |dir: &Path| entries.iter().position(|entry| entry == dir); + assert!( + position(&sdk_lib).unwrap() < position(&compat_dir).unwrap(), + "the link directory must come after the SDK's: {entries:?}" + ); + } + + /// The amd-smi link puts the compat directory on the loader path even when the + /// host has a real `libmpi_cxx`, so the launch removes a stub an earlier run + /// left there. With no real `libmpi_cxx` on the host the stub may be the one + /// torch is loading and stays. + /// + /// Which of the two holds depends on the host, so the expectation is derived + /// from the probe the launch uses, and only the half that matches this host + /// runs here. Both halves, with the host answer injected, are tested in + /// rocm-core (`the_shared_helpers_clear_a_leftover_mpi_cxx_stub_only_beside_a_real_library`); + /// this test pins that the launch goes through that helper. + #[cfg(target_os = "linux")] + #[test] + fn a_launch_clears_a_leftover_mpi_cxx_stub_only_beside_a_real_libmpi_cxx() { + let env = tempfile::tempdir().unwrap(); + let sdk_root = env.path().join("sdk"); + let sdk_lib = sdk_root.join("lib"); + fs::create_dir_all(&sdk_lib).unwrap(); + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk").unwrap(); + let compat_dir = env.path().join("rocm-cli-lib-compat"); + fs::create_dir_all(&compat_dir).unwrap(); + let stub = compat_dir.join(rocm_core::openmpi::MPI_CXX_STUB_NAME); + fs::write(&stub, b"stub").unwrap(); + let runtime = VllmRuntime { + runtime_id: "release-wheel".to_owned(), + env_id: "release-wheel".to_owned(), + python_executable: Some(env.path().join("bin").join("python")), + sdk_root: Some(sdk_root), + sdk_library_paths: vec![sdk_lib], + ..VllmRuntime::for_test() + }; + + // `ensure_mpi_cxx_compat` answers `None`, whatever the reason. + let shim = ensure_therock_compat_shims(&runtime, |_| None); + + assert_eq!(shim.as_deref(), Some(compat_dir.as_path())); + assert!(compat_dir.join("libamd_smi.so").exists()); + assert_eq!( + stub.exists(), + !rocm_core::openmpi::real_libmpi_cxx_present() + ); + } + + /// The "a bare name already exists" rule reads the SDK's directories only, not + /// the compat directory itself: a link there left dangling or pointing at + /// another SDK is repaired, not taken for the SDK's own bare name. + #[cfg(target_os = "linux")] + #[test] + fn an_existing_amd_smi_link_in_the_compat_dir_is_repaired_not_trusted() { + let env = tempfile::tempdir().unwrap(); + let sdk_root = env.path().join("sdk"); + let sdk_lib = sdk_root.join("lib"); + fs::create_dir_all(&sdk_lib).unwrap(); + fs::write(sdk_lib.join("libamd_smi.so.26"), b"sdk").unwrap(); + let other = env.path().join("other-sdk"); + fs::create_dir_all(&other).unwrap(); + fs::write(other.join("libamd_smi.so.25"), b"other").unwrap(); + let compat_dir = env.path().join("rocm-cli-lib-compat"); + fs::create_dir_all(&compat_dir).unwrap(); + let link = compat_dir.join("libamd_smi.so"); + let runtime = VllmRuntime { + runtime_id: "release-wheel".to_owned(), + env_id: "release-wheel".to_owned(), + python_executable: Some(env.path().join("bin").join("python")), + sdk_root: Some(sdk_root), + sdk_library_paths: vec![sdk_lib.clone()], + ..VllmRuntime::for_test() + }; + + for stale_target in [env.path().join("gone"), other.join("libamd_smi.so.25")] { + let _ = fs::remove_file(&link); + std::os::unix::fs::symlink(&stale_target, &link).unwrap(); + + let shim = ensure_therock_compat_shims(&runtime, |_| None); + + assert_eq!(shim.as_deref(), Some(compat_dir.as_path())); + assert_eq!( + fs::read_link(&link).unwrap(), + sdk_lib.join("libamd_smi.so.26"), + "a link to {} must be repointed", + stale_target.display() + ); + } + } + #[test] fn load_runtime_manifests_breaks_install_time_ties_on_the_runtime_key() -> Result<()> { // Both pairings of file name and runtime key, so a read-order pick is wrong diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index bf2fce5385..7e87aa8930 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -191,8 +191,10 @@ pub(crate) fn read_service_state(path: &Path) -> Result { /// [`write_terminal_state`] both go through — it is not the only way this /// module's state files are written: inference-verification latches go /// through `rocm_core::merge_json_state_file` instead (see -/// [`inference_verified`]), which patches the file directly rather than -/// routing through here. +/// [`inference_verified`]), which patches the file under the state +/// directory's lock rather than routing through here. This function takes no +/// such lock and writes in place, so the lock does not order it against those +/// latches. fn write_state(path: &Path, value: &Value) -> Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) diff --git a/engines/vllm/src/wheel_builds.rs b/engines/vllm/src/wheel_builds.rs index 6334eb7085..6c3464a128 100644 --- a/engines/vllm/src/wheel_builds.rs +++ b/engines/vllm/src/wheel_builds.rs @@ -332,7 +332,7 @@ mod tests { // with no row still fails closed instead of taking the 7.2.3 default // pin; other lines (and unparsable text) resolve that default. assert_eq!( - discover_readers(Some("10.2.0")), + discover_readers(Some("10.3.0")), DiscoverReaders { row: None, route: VllmInstallRoute::Static, @@ -540,6 +540,7 @@ mod tests { Some("2.11.0.2") ); assert_eq!(verified_torchaudio_release("10.0.0"), None); + // The 10.2 row's torchaudio is a series match, hardware-unverified. assert_eq!(verified_torchaudio_release("10.2.0"), None); assert_eq!(verified_torchaudio_release("7.2.3"), None); } @@ -559,7 +560,11 @@ mod tests { verified_torch_stack_series("10.0.0"), Some(("2.12", "0.27")) ); - assert_eq!(verified_torch_stack_series("10.2.0"), None); + assert_eq!( + verified_torch_stack_series("10.2.0"), + Some(("2.14", "0.29")) + ); + assert_eq!(verified_torch_stack_series("10.3.0"), None); assert_eq!(verified_torch_stack_series("7.2.3"), None); } /// An unknown 10.x line fails closed rather than silently resolving the @@ -567,7 +572,7 @@ mod tests { /// [`rocm_sdk_major_matches`] alongside the narrower row lookup. #[test] fn unknown_rocm10_line_still_fails_closed() { - let err = install_target(None, Some("10.2.0")).expect_err("10.2.0 has no vLLM build"); + let err = install_target(None, Some("10.3.0")).expect_err("10.3.0 has no vLLM build"); let rendered = format!("{err:#}"); assert!( !rendered.contains("7.2.3"), diff --git a/engines/vllm/tests/stdio_binary.rs b/engines/vllm/tests/stdio_binary.rs new file mode 100644 index 0000000000..e499504f67 --- /dev/null +++ b/engines/vllm/tests/stdio_binary.rs @@ -0,0 +1,67 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `rocm-engine-vllm stdio` binary, end to end: what it reads on stdin goes +//! through the shared request reader, so a request it cannot serve, or input +//! that is not a request, is answered with an error envelope on stdout and a +//! clean exit. + +use std::io::Write as _; +use std::process::{Command, Stdio}; + +/// Run `rocm-engine-vllm stdio` with `request` on stdin, on scratch app roots, +/// and return its exit status and the error code of the envelope it printed. +fn answer(label: &str, request: &str) -> (Option, String) { + let root = std::env::temp_dir().join(format!( + "rocm-engine-vllm-stdio-{label}-{}", + std::process::id() + )); + let mut child = Command::new(env!("CARGO_BIN_EXE_rocm-engine-vllm")) + .arg("stdio") + .env("ROCM_CLI_CONFIG_DIR", root.join("config")) + .env("ROCM_CLI_DATA_DIR", root.join("data")) + .env("ROCM_CLI_CACHE_DIR", root.join("cache")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start the engine binary"); + child + .stdin + .take() + .expect("piped stdin") + .write_all(request.as_bytes()) + .expect("write the request"); + let output = child + .wait_with_output() + .expect("wait for the engine binary"); + let _ = std::fs::remove_dir_all(&root); + let stdout = String::from_utf8_lossy(&output.stdout); + let envelope: serde_json::Value = serde_json::from_str(&stdout).unwrap_or_else(|error| { + panic!( + "{label}: no envelope on stdout ({error}): {stdout:?}; stderr: {}", + String::from_utf8_lossy(&output.stderr) + ) + }); + let code = envelope["error"]["code"] + .as_str() + .unwrap_or_default() + .to_owned(); + (output.status.code(), code) +} + +#[test] +fn the_stdio_binary_answers_requests_it_cannot_serve() { + assert_eq!( + answer( + "incompatible", + r#"{"protocol_version":"9.0.0","method":"frobnicate","payload":{}}"# + ), + (Some(0), "incompatible_protocol".to_owned()) + ); + assert_eq!( + answer("garbage", "this is not a request"), + (Some(0), "invalid_request".to_owned()) + ); +} diff --git a/install.ps1 b/install.ps1 index 39bba211e7..c1c9da0c91 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,3 +1,7 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + param( [Parameter(Position = 0)] [string] $Channel = $env:ROCM_CLI_CHANNEL, @@ -168,14 +172,18 @@ function Convert-FileUriToPath { return $null } -# Pinned production release signing public keys (trust roots). These stay empty -# until the repository owner publishes production keys (see docs/release-trust.md, -# "Remaining Owner Step"). While empty, release installs keep the opt-in behavior: -# a signature is verified only when a key is supplied via the parameters/env vars -# or ROCM_CLI_REQUIRE_SIGNATURE=1. Once populated, release-channel installs verify -# signatures by default with these keys as trust roots. Two slots support -# zero-downtime key rotation: a release signed with either the current or the -# pre-staged next key verifies, so the next key is trusted before its first use. +# Pinned production release signing public keys (trust roots). The Current slot +# holds the published release key, whose canonical copy is +# docs/keys/rocm-cli-release-current-public.pem; the Next slot stays "" until a +# rotation key is published. Resolve-SigningPublicKey falls back to these keys +# whenever no key is passed with -SigningPublicKeyPath/-SigningPublicKeyPem (or +# ROCM_CLI_SIGNING_PUBLIC_KEY_PATH/ROCM_CLI_SIGNING_PUBLIC_KEY_PEM), so while a +# release key is pinned every install on any channel verifies the detached +# signature and a missing .sig sidecar fails it, whether +# ROCM_CLI_REQUIRE_SIGNATURE is unset or 0. A key supplied that way replaces +# these keys rather than adding to them. Two slots support zero-downtime key +# rotation: a release signed with either the current or the pre-staged next key +# verifies, so the next key is trusted before its first use. $PinnedReleasePublicKeyCurrent = @" -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxuyScR/BzV+kuXqWAHtE @@ -724,10 +732,16 @@ $sigPath = "$archivePath.sig" $extractDir = Join-Path $tempRoot "extract" $manifestPath = Join-Path $InstallDir ".rocm-cli-manifest" $signatureRequired = $RequireSignature -or (Test-Truthy $env:ROCM_CLI_REQUIRE_SIGNATURE) -# Production default: once release trust roots are pinned, release-channel installs -# always verify. An unset or 0 ROCM_CLI_REQUIRE_SIGNATURE does not lower this -# floor; pass -SigningPublicKeyPath/-Pem (or the env vars) to point at an -# alternate key (e.g. a private mirror) instead. +# This floor changes no outcome while a release key is pinned: +# Resolve-SigningPublicKey then always returns a key, and that alone makes the +# check below verify. If both pinned slots were emptied, +# Test-HasPinnedReleaseKey is false and the floor does not fire either, so a +# release install with no key supplied would skip verification, unless +# -RequireSignature or ROCM_CLI_REQUIRE_SIGNATURE=1 is given, which makes it +# fail for want of a key. The floor is left as it is on purpose: making a release +# install fail when no key resolves is tracked as ROCMAI-960. Pass +# -SigningPublicKeyPath/-Pem (or the env vars) to point at an alternate key +# (e.g. a private mirror). if ($Channel -eq "release" -and (Test-HasPinnedReleaseKey)) { $signatureRequired = $true } diff --git a/install.sh b/install.sh index 01d5820edb..26627a252d 100755 --- a/install.sh +++ b/install.sh @@ -70,13 +70,85 @@ ok() { printf '%s\n' "${C_OK}::${C_RESET} $*" } +# Width to draw a download progress bar at, or empty when no bar should be +# drawn. curl sizes its bar from COLUMNS whenever that variable parses to a +# sane number, in preference to asking the terminal, and then draws a line +# exactly that wide. A COLUMNS left over from a wider window, routine when a +# terminal is resized after the shell starts, therefore makes every refresh run +# past the right margin and wrap; the carriage return ending each frame then +# returns to the start of the wrapped row rather than the bar's, so each redraw +# paints a row lower and one download arrives as a stack of partial bars with +# its percentage torn across two rows. Measuring the terminal here and passing +# that width down puts the real one back in charge, and reserving the last +# column keeps the line clear of the margin if the measurement is off by one. +progress_bar_width() { + width="" + if command -v stty >/dev/null 2>&1; then + # fd 2 is what fetch() gates the bar on, and the only descriptor still + # attached to the terminal under `curl ... | sh`, where stdin is the pipe. + width="$(stty size <&2 2>/dev/null | awk '{print $2}')" + fi + # A terminal whose window size was never set (a freshly opened pty, say) + # reports 0. That means "unmeasured", not "extremely narrow", so fall through + # to the other sources rather than treating it as a width and dropping the + # bar that curl would otherwise have drawn at its own default. + [ "${width}" = "0" ] && width="" + if [ -z "${width}" ] && command -v tput >/dev/null 2>&1; then + # Read fd 2 here too: tput otherwise sizes from its own stdin, which under + # `curl ... | sh` is the pipe, making ncurses report its 80-column default + # instead of the terminal. COLUMNS is blanked because ncurses consults it + # first, and would hand back the stale value this function exists to escape. + width="$(COLUMNS='' tput cols <&2 2>/dev/null || true)" + fi + [ "${width}" = "0" ] && width="" + if [ -z "${width}" ]; then + width="${COLUMNS:-80}" + fi + case "${width}" in + "" | *[!0-9]*) width=80 ;; + *) : ;; + esac + # Strip leading zeros. A zero-padded value such as 080 passes the digit check + # above but is an octal literal to POSIX arithmetic, which dash rejects + # outright ("Illegal number: 080"). bash's 10# prefix would force base 10 but + # is not POSIX, and dash rejects that too, so do it by hand. + while :; do + case "${width}" in + 0[0-9]*) width="${width#0}" ;; + *) break ;; + esac + done + # A bare 0 reaching here came from the environment rather than a measurement, + # so treat it as unmeasured like the terminal sources above. + [ "${width}" = "0" ] && width=80 + # curl ignores a COLUMNS of 20 or less and sizes the bar itself, which would + # reintroduce the overrun this function exists to prevent. Report no width so + # the caller downloads quietly instead. + if [ "${width}" -le 21 ]; then + return 0 + fi + printf '%s\n' "$((width - 1))" +} + +# Set once the narrow-terminal notice below has been printed, so three +# downloads do not repeat it. +narrow_terminal_reported=0 + fetch() { url="$1" output="$2" failure_message="${3:-failed to download ${url}}" if command -v curl >/dev/null 2>&1; then + bar_width="" if [ -t 2 ]; then - curl -fL --progress-bar "${url}" -o "${output}" || { + bar_width="$(progress_bar_width)" + if [ -z "${bar_width}" ] && [ "${narrow_terminal_reported}" -eq 0 ]; then + info "terminal too narrow for a progress bar, downloading quietly" + narrow_terminal_reported=1 + fi + fi + if [ -n "${bar_width}" ]; then + COLUMNS="${bar_width}" curl -fL --progress-bar "${url}" -o "${output}" || { rm -f "${output}" fail "${failure_message}" } @@ -88,7 +160,12 @@ fetch() { fi elif command -v wget >/dev/null 2>&1; then if [ -t 2 ]; then - wget --show-progress -O "${output}" "${url}" || { + # --show-progress alone leaves wget's full request logging in place, so + # the bar is interleaved with resolver and redirect chatter. -nv drops + # that chatter and keeps the bar, which wget already holds inside the + # terminal width. Not -q, which would also swallow wget's reason for a + # failure and leave only this script's generic message. + wget -nv --show-progress -O "${output}" "${url}" || { rm -f "${output}" fail "${failure_message}" } @@ -103,12 +180,15 @@ fetch() { fi } -# Pinned production release signing public keys (trust roots). These stay empty -# until the repository owner publishes production keys (see docs/release-trust.md, -# "Remaining Owner Step"). While empty, release installs keep the opt-in behavior -# below: a signature is verified only when a key is supplied via the env vars or -# ROCM_CLI_REQUIRE_SIGNATURE=1. Once populated, release-channel installs verify -# signatures by default with these keys as trust roots. Two slots support +# Pinned production release signing public keys (trust roots). The CURRENT slot +# holds the published release key, whose canonical copy is +# docs/keys/rocm-cli-release-current-public.pem; the NEXT slot stays "" until a +# rotation key is published. resolve_public_keys falls back to these keys +# whenever ROCM_CLI_SIGNING_PUBLIC_KEY_PATH and ROCM_CLI_SIGNING_PUBLIC_KEY_PEM +# are unset, so while a release key is pinned every install on any channel +# verifies the detached signature and a missing .sig sidecar fails it, whether +# ROCM_CLI_REQUIRE_SIGNATURE is unset or 0. A key supplied through those env +# vars replaces these keys rather than adding to them. Two slots support # zero-downtime key rotation: a release signed with either the current or the # pre-staged next key verifies, so the next key is trusted before its first use. PINNED_RELEASE_PUBLIC_KEY_CURRENT="-----BEGIN PUBLIC KEY----- @@ -626,10 +706,15 @@ require_sig=0 if truthy "${ROCM_CLI_REQUIRE_SIGNATURE:-0}"; then require_sig=1 fi -# Production default: once release trust roots are pinned, release-channel -# installs always verify. An unset or 0 ROCM_CLI_REQUIRE_SIGNATURE does not lower -# this floor; use ROCM_CLI_SIGNING_PUBLIC_KEY_PATH/PEM to point at an alternate -# key (e.g. a private mirror) instead. +# This floor changes no outcome while a release key is pinned: +# resolve_public_keys then always returns a key, and that alone makes the check +# below verify. If both pinned slots were emptied, has_pinned_release_keys is +# false and the floor does not fire either, so a release install with no key +# supplied would skip verification, unless ROCM_CLI_REQUIRE_SIGNATURE=1, which +# makes it fail for want of a key. The floor is left as it is on purpose: making +# a release install fail when no key resolves is tracked as ROCMAI-960. Use +# ROCM_CLI_SIGNING_PUBLIC_KEY_PATH/PEM to point at an alternate key (e.g. a +# private mirror). if [ "${CHANNEL}" = "release" ] && has_pinned_release_keys; then require_sig=1 fi diff --git a/licenserc.toml b/licenserc.toml index 161104918c..da366676d0 100644 --- a/licenserc.toml +++ b/licenserc.toml @@ -15,9 +15,17 @@ ## hawkeye does not detect it below it. Each file carries the header directly ## after its front matter. Add any new page that needs front matter here. +## The header text is a template rendered per file. PowerShell files get an ASCII +## `(c)` instead of the copyright sign: a file with any non-ASCII byte needs a +## BOM, and install.ps1 cannot have one because a BOM breaks `irm ... | iex`. The +## other PowerShell files follow so the whole family reads the same. [header] text = """ +{% if attrs.filename is endingwith(".ps1") or attrs.filename is endingwith(".psm1") or attrs.filename is endingwith(".psd1") -%} +Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +{%- else -%} Copyright © Advanced Micro Devices, Inc., or its affiliates. +{%- endif %} SPDX-License-Identifier: MIT """ @@ -28,6 +36,9 @@ includes = [ "**/*.py", "**/*.sh", "**/*.md", + "**/*.ps1", + "**/*.psm1", + "**/*.psd1", ] excludes = [ @@ -45,3 +56,12 @@ excludes = [ [[rules]] extensions = ["md"] style_out = "xml" + +## hawkeye has no built-in comment syntax for PowerShell: without this rule a +## .ps1/.psm1/.psd1 file is reported `unsupported` and silently skipped (exit 0), +## so `includes` alone would not enforce anything. Only `hawkeye check +## --fail-on-unknown` turns that into a failure, and neither the CI job nor the +## prek hook passes the flag. +[[rules]] +extensions = ["ps1", "psm1", "psd1"] +style_out = "script" diff --git a/ruff.toml b/ruff.toml index 55c3eda733..8af4ab98bc 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -# Ruff configuration for the Python helper scripts (scripts/) and engine workers. +# Ruff configuration for every Python file in the repository. # Formatting uses Ruff's defaults (88-column, double quotes), which the # `ruff-format` pre-commit hook enforces. @@ -15,7 +15,44 @@ select = [ "SIM", # flake8-simplify "C4", # flake8-comprehensions "RUF", # Ruff-specific rules + "ARG", # flake8-unused-arguments + "BLE", # flake8-blind-except + "SLF", # flake8-self (private-member access) + "PL", # Pylint (PLC/PLE/PLR/PLW) ] ignore = [ "E501", # line length is enforced by the formatter; long comments/URLs are fine + # + # Pylint refactor metrics, ignored for every file: acting on them means + # restructuring existing scripts, which enabling the rule families does not do. + "PLR0912", # too-many-branches + "PLR0913", # too-many-arguments + "PLR0915", # too-many-statements + # + # Not a refactor metric: it flags inline numeric literals in comparisons, and + # replacing them with named constants is a separate cleanup. + "PLR2004", # magic-value-comparison ] + +[lint.per-file-ignores] +# A waived rule checks nothing anywhere in that file, so new code in these files +# is reviewed by hand against the rule stated on the entry. +# +# A blind `except` must report the failure, fall back, or keep the error for a +# later message. +"scripts/comfyui_therock_gpu_test.py" = ["BLE001"] # blind-except +"scripts/therock_sdk_install_test.py" = ["BLE001"] # blind-except +"scripts/vllm_therock_gpu_test.py" = ["BLE001"] # blind-except +# Function-level imports belong to the self-test only; a module global rebound +# here is restored in a `finally`. +"scripts/vllm_route_check.py" = ["PLC0415", "PLW0603"] # import-outside-top-level + global-statement +# The skills/rocm-doctor/ entries stay waived rather than edited: a change there on +# main opens a "Bump rocm-doctor" pull request in amd/skills through its nightly +# workflow (see the `skills/rocm-doctor/` part of AGENTS.md). +# +# ARG001 covers only the unused `workspace` and `case` parameters of setup() and +# teardown(); any other unused argument in the file is removed. `_active` is the +# only module global the hooks rebind. +"skills/rocm-doctor/evals/hooks.py" = ["ARG001", "PLW0603"] # unused-function-argument + global-statement +# A `subprocess.run` here has its returncode asserted by the test. +"skills/rocm-doctor/evals/test_hooks.py" = ["PLW1510"] # subprocess-run-without-check diff --git a/runtime-deps.toml b/runtime-deps.toml index 44885660bf..54fbb39814 100644 --- a/runtime-deps.toml +++ b/runtime-deps.toml @@ -3,8 +3,11 @@ # This file is the SINGLE SOURCE OF TRUTH for those pins. Nothing else in the # tree may spell a runtime version literal: archive names, download URLs, and # offline fallbacks are all derived from the values below by the `rocm-deps` -# crate, whose build script turns each entry into a Rust constant. Editing a -# version here is the whole bump. +# crate, whose build script turns each entry into a Rust constant. A bump also +# records the new archives' digests where the adapter verifies them; for +# Lemonade, that is `EMBEDDABLE_*_SHA256` in engines/lemonade/src/install.rs, +# together with what the new Linux `lemond` needs from the host +# (`LINUX_LEMOND_NEEDS` beside them). # # Schema: one `[runtime.]` table per managed runtime, with one or more # string fields. Every field becomes a `pub const _: &str` in @@ -15,4 +18,4 @@ # Lemonade embeddable release, published as # `lemonade-embeddable--.{tar.gz|zip}` under the # `v` tag of the lemonade-sdk/lemonade GitHub releases. -version = "11.5.1" +version = "2026.44.1" diff --git a/scripts/local_assistant_therock_gpu_test.py b/scripts/local_assistant_therock_gpu_test.py index 47769721f9..bf1728d9ff 100644 --- a/scripts/local_assistant_therock_gpu_test.py +++ b/scripts/local_assistant_therock_gpu_test.py @@ -68,7 +68,7 @@ def main() -> int: print_step(f"Copied ROCm runtime registry from {source_state}.") localize_huggingface_cache(env, repo_root) - return run_live_acceptance(args, rocm, env, temp_state_root) + return run_live_acceptance(args, rocm, env) finally: if temp_state_root is not None and not args.keep_state: print_step(f"Removing temporary ROCm CLI state under {temp_state_root}.") @@ -79,7 +79,6 @@ def run_live_acceptance( args: argparse.Namespace, rocm: Path, env: dict[str, str], - temp_state_root: Path | None, ) -> int: data_dir = rocm_cli_data_dir(env) service_id = args.service_id diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 68e8569f84..4e2ee32a3b 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -22,6 +22,11 @@ # both (default "release nightly") # E2E_LANE_WORK where the build and runtimes go (default /var/tmp/e2e-lane) # LANE_OUT where reported files go (default /out) +# E2E_EXPECT_GPU whether the lane has a GPU (default 1: every queue lane is +# given one); the suite fails rather than passing with every +# GPU scenario skipped. Set but empty, it leaves the lane +# undeclared, which switches that check off. See +# tests/e2e-cucumber/README.md. set -euo pipefail @@ -39,6 +44,7 @@ if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then fi export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" +export E2E_EXPECT_GPU="${E2E_EXPECT_GPU-1}" export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" # The channels are checked before anything is built, so a typo costs nothing. diff --git a/scripts/vllm_route_check.py b/scripts/vllm_route_check.py new file mode 100644 index 0000000000..98c1dc675a --- /dev/null +++ b/scripts/vllm_route_check.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +"""Check that vLLM has an install route for the SDKs an unpinned install picks. + +Runs ``rocm install sdk --channel C --family F --dry-run`` for the release and +the nightly channel on each Instinct arch (the targets whose default engine is +vLLM) and reports a finding when the ROCm SDK version the plan selects has neither a +discovery row nor a static pin in the vLLM engine's tables. Without a route, +``rocm engines install vllm`` refuses the runtime, and every vLLM-default GPU +lane runs no scenarios on it (the nightly queue lanes did exactly that on the +10.3.0a nightly SDKs). + +The tables are read from the engine's own source, so this cannot drift from +them, and the row matching is the engine's: a discovery row matches on +``major.minor``, a static pin on ``major.minor.patch``, both ignoring any +``aYYYYMMDD`` suffix. It is stricter than the engine in one place on purpose: +a version with no row whose major has no discovery row makes the engine fall +back to the default static pin (a ROCm 7.2.3 vLLM wheel) rather than refuse, +and this reports that too, because that wheel is not built for the SDK. +Nothing is installed and no GPU is needed. + +It resolves the live indexes, so it can report a gap with no change in the PR: +an index that moved on to an SDK vLLM has no wheels for. A finding exits 1 and +the workflow that runs it turns that into a warning, for that reason. Add the +missing row, or wait for the wheels. A check that could not run (a dry run that +times out, fails or prints no usable version, an unreadable table, a missing +``rocm``) exits 2, which fails the job. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +DISCOVER_FILE = Path("engines/vllm/src/discover_builds.rs") +STATIC_FILE = Path("engines/vllm/src/wheel_builds.rs") +CHANNELS = ("release", "nightly") +# The next-layout Instinct arches. Not every vLLM-default arch is here (see +# crates/rocm-core/src/host_gpu/engine_default.rs): add one by hand, or it is +# never probed. +# Exact arches, because the next layout selects its device payload by arch and +# refuses a grouped family. gfx942 is MI300X/MI325X, gfx950 is MI350/MI355. +FAMILIES = ("gfx942", "gfx950") +DRY_RUN_TIMEOUT_SECS = 300 + + +class CheckError(Exception): + """The check could not run to a verdict.""" + + +def read_routes(discover_text: str, static_text: str) -> dict[str, set]: + """The ROCm SDK lines the vLLM engine installs for, from its two tables. + + ``discover`` holds ``(major, minor)`` lines and ``static`` holds + ``(major, minor, patch)`` releases. Only the code above each file's test + module is read, so a test fixture cannot count as a route. + """ + + def literals(text: str) -> list[tuple[int, ...]]: + text = text.split("#[cfg(test)]", 1)[0] + return [ + tuple(int(part) for part in found.split(".")) + for found in re.findall(r'rocm_sdk_version:\s*"(\d+\.\d+\.\d+)"', text) + ] + + discover = {version[:2] for version in literals(discover_text)} + static = set(literals(static_text)) + if not discover or not static: + raise CheckError( + "could not read the vLLM build tables; VLLM_ROCM_DISCOVER_BUILD_TABLE " + "or VLLM_ROCM_BUILD_TABLE was renamed or moved" + ) + return {"discover": discover, "static": static} + + +def route_problem(version: str, routes: dict[str, set]) -> str | None: + """Why vLLM has no build for ``version``; ``None`` when it has one.""" + found = re.match(r"^(\d+)\.(\d+)\.(\d+)", version) + if not found: + return f"selected version {version!r} is not a ROCm release version" + triple = tuple(int(part) for part in found.groups()) + if triple[:2] in routes["discover"] or triple in routes["static"]: + return None + outcome = ( + "`rocm engines install vllm` refuses that runtime" + if triple[0] in {line[0] for line in routes["discover"]} + else "`rocm engines install vllm` would fall back to the default static pin, " + "a wheel built for another ROCm" + ) + return ( + f"vLLM has no build for ROCm SDK {version}: no discovery row for " + f"{triple[0]}.{triple[1]} and no static pin for " + f"{triple[0]}.{triple[1]}.{triple[2]}, so {outcome}" + ) + + +def selected_version(output: str) -> str | None: + """The ``selected_rocm_version`` line of a dry-run plan.""" + for line in output.splitlines(): + match = re.match(r"^\s+selected_rocm_version: (\S+)\s*$", line) + if match: + return match.group(1) + return None + + +def probe(rocm: str, channel: str, family: str, routes: dict[str, set]) -> str | None: + """Resolve one channel and family; return a missing-route finding, or ``None``. + + Only a real "no vLLM route" finding is returned (the caller reports it as + exit 1). A dry run that times out, fails, or prints no selected version + means the check itself broke, which is no verdict on routing, so it raises + ``CheckError`` and the run exits 2. + """ + label = f"{channel}/{family}" + command = [ + rocm, "install", "sdk", "--channel", channel, "--family", family, + "--dry-run", + ] # fmt: skip + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=DRY_RUN_TIMEOUT_SECS, + check=False, + ) + except OSError as error: + raise CheckError(f"{label}: cannot run {rocm}: {error}") from error + except subprocess.TimeoutExpired: + raise CheckError( + f"{label}: dry run did not finish in {DRY_RUN_TIMEOUT_SECS}s" + ) from None + if result.returncode != 0: + tail = (result.stderr or result.stdout).strip().splitlines()[-3:] + raise CheckError( + f"{label}: dry run failed ({result.returncode}): {' | '.join(tail)}" + ) + version = selected_version(result.stdout) + if version is None: + raise CheckError(f"{label}: the plan has no selected_rocm_version line") + if not re.match(r"^\d+\.\d+\.\d+", version): + raise CheckError(f"{label}: unparseable selected_rocm_version {version!r}") + problem = route_problem(version, routes) + if problem: + return f"{label}: {problem}" + print(f"{label}: ROCm SDK {version} has a vLLM build") + return None + + +SAMPLE_DISCOVER = """ + VllmRocmDiscoverBuild { + rocm_sdk_version: "10.0.0", + }, + VllmRocmDiscoverBuild { + rocm_sdk_version: "10.1.0", + }, +#[cfg(test)] + rocm_sdk_version: "10.9.0", +""" +SAMPLE_STATIC = """ +const VLLM_ROCM_BUILD_TABLE: &[VllmRocmBuild] = &[VllmRocmBuild { + rocm_sdk_version: "7.2.3", +}]; +""" + + +def probe_self_test(routes: dict[str, set]) -> None: + """``probe`` splits "no route" (a return) from "the check broke" (a raise).""" + import os + import stat + import tempfile + + def fake_rocm(directory: str, body: str) -> str: + path = os.path.join(directory, "rocm") + with open(path, "w", encoding="utf-8") as handle: + handle.write(f"#!{sys.executable}\n{body}\n") + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + return path + + plan = "print(' selected_rocm_version: {}')" + with tempfile.TemporaryDirectory() as directory: + ok = fake_rocm(directory, plan.format("10.1.0")) + assert probe(ok, "nightly", "gfx942", routes) is None + gap = fake_rocm(directory, plan.format("10.3.0a20261101")) + found = probe(gap, "nightly", "gfx942", routes) + assert found and "refuses" in found, found + broken_runs = ( + ("import sys; sys.exit(3)", 300), + (plan.format("10.1.0") + "; import sys; sys.exit(3)", 300), + ("print('no version here')", 300), + (plan.format("nightly"), 300), + ("import time; time.sleep(30)", 1), + ) + global DRY_RUN_TIMEOUT_SECS + saved = DRY_RUN_TIMEOUT_SECS + try: + for body, timeout in broken_runs: + DRY_RUN_TIMEOUT_SECS = timeout + broken = fake_rocm(directory, body) + try: + probe(broken, "nightly", "gfx942", routes) + except CheckError: + pass + else: + raise AssertionError(f"a broken dry run must raise: {body}") + finally: + DRY_RUN_TIMEOUT_SECS = saved + try: + probe(os.path.join(directory, "missing"), "nightly", "gfx942", routes) + except CheckError: + pass + else: + raise AssertionError("a missing rocm must raise, not crash") + + +def self_test() -> None: + """Each rule must reject what it exists for, and accept a covered version.""" + routes = read_routes(SAMPLE_DISCOVER, SAMPLE_STATIC) + assert routes == {"discover": {(10, 0), (10, 1)}, "static": {(7, 2, 3)}}, routes + assert route_problem("10.1.0", routes) is None + assert route_problem("10.1.0a20261101", routes) is None, "suffix is ignored" + assert route_problem("10.1.7", routes) is None, "a discovery row spans patches" + assert route_problem("7.2.3a20260423", routes) is None + assert route_problem("7.2.4", routes) is not None, "a static pin is exact" + refused = route_problem("10.3.0", routes) + assert refused and "refuses" in refused, refused + fallback = route_problem("11.0.0", routes) + assert fallback and "fall back" in fallback, fallback + assert route_problem("10.9.0", routes) is not None, "test fixtures are no route" + new_line = route_problem("10.3.0a20261115", routes) + assert new_line and "10.3" in new_line and "refuses" in new_line, new_line + assert route_problem("nightly", routes) is not None + for broken in (("", SAMPLE_STATIC), (SAMPLE_DISCOVER, "")): + try: + read_routes(*broken) + except CheckError: + pass + else: + raise AssertionError("a renamed table must fail loudly, not pass") + plan = "sdk install\n channel: nightly\n selected_rocm_version: 10.3.0a20261101\n" + assert selected_version(plan) == "10.3.0a20261101" + assert selected_version("sdk install\n") is None + probe_self_test(routes) + print("vllm route check self-test: ok") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--rocm", help="path to the rocm binary to probe") + parser.add_argument("--discover-file", type=Path, default=DISCOVER_FILE) + parser.add_argument("--static-file", type=Path, default=STATIC_FILE) + parser.add_argument("--channel", action="append", dest="channels") + parser.add_argument("--family", action="append", dest="families") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + return 0 + if not args.rocm: + parser.error("--rocm is required unless --self-test is given") + try: + routes = read_routes( + args.discover_file.read_text(encoding="utf-8"), + args.static_file.read_text(encoding="utf-8"), + ) + except (OSError, UnicodeDecodeError) as error: + raise CheckError(f"cannot read the engine's build tables: {error}") from error + failures = [] + for channel in args.channels or CHANNELS: + for family in args.families or FAMILIES: + failure = probe(args.rocm, channel, family, routes) + if failure: + print(f"::warning::{failure}") + failures.append(failure) + return 1 if failures else 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckError as error: + print(f"::error::{error}") + sys.exit(2) diff --git a/skills/rocm-doctor/SKILL.md b/skills/rocm-doctor/SKILL.md index 85a8df9543..7af50732ce 100644 --- a/skills/rocm-doctor/SKILL.md +++ b/skills/rocm-doctor/SKILL.md @@ -235,7 +235,7 @@ passes — the GPU is AMD or no vendor was named. ``` rocm fix # prompts before changing anything - rocm fix --dry-run # show the exact change, touch nothing + rocm fix --dry-run # show the exact change, touch nothing (see the Windows exit 4 below) rocm fix --yes # required to apply in a non-interactive shell ``` @@ -264,6 +264,53 @@ passes — the GPU is AMD or no vendor was named. pinned. Once the user knows N, re-run with `rocm fix fix-9-igpu-dgpu --device-index N`. + If `rocm fix` exits 4, relay its stderr line and act on how it ends: + + - `Could not read the current …; nothing was changed.` — on Windows, + `fix-2-unset-override`, `fix-6-path` and `fix-9-igpu-dgpu --device-index N` + read the variable they change before planning, and that read failed + (`--dry-run` included). Tell the user nothing was changed and relay the + cause the line states; once it is cleared (for example PowerShell starts + and answers in time), offer to re-run the same `rocm fix` command. Do not + present the printed catalog entry as a preview, and do not hand the user + its commands (such as `setx HIP_VISIBLE_DEVICES `) instead. + - `… NOT changed.` — the command ran and failed, or could not start. + Nothing was changed, unless the line says `exited 124` or `exited -1` + (see below). + - `… may or may not have changed. Check … then re-run …` — the command + timed out or was killed. Relay the check (on Windows, an expression to run + in PowerShell) and the `rocm fix` command to re-run; claim neither that + the fix was applied nor that nothing changed until the check has run. + - `Failed to write : …` — the rc-file append failed; have the user + look at that file. + + Some CLIs that pass the Phase 0 contract check are older than these lines + and report two cases differently. The second looks the same as a current + CLI's output, so it applies to every Windows fix-2 and fix-9 run: + + - A `NOT changed` line saying `exited 124` (timed out) or `exited -1` + (killed by a signal) does not prove nothing changed: the change may or + may not have landed. Say so, have the user check (`id -nG ` for + `fix-4-render-group`; on Windows, in PowerShell, + `[Environment]::GetEnvironmentVariable('','User')` with `` + being `HSA_OVERRIDE_GFX_VERSION`, `PATH` or `HIP_VISIBLE_DEVICES` for + fix-2, fix-6 or fix-9), and offer to re-run the same `rocm fix` command + if it did not land. + - On Windows such a CLI takes a failed read for an unset variable, so a + clean result is not proof. `fix-2-unset-override` can say + `No persistent HSA_OVERRIDE_GFX_VERSION found` and that the user is done, + and exit 0, with the override still set in either scope; + `fix-9-igpu-dgpu --device-index N` can run `setx` over a value the user + set on purpose. So have the user run that PowerShell check. For fix-2, + run it before calling the fix done, with `'User'` and again with + `'Machine'`; blank output means unset. A User value means re-running + `rocm fix fix-2-unset-override`; a Machine value has to be cleared from + an elevated PowerShell with + `[Environment]::SetEnvironmentVariable('HSA_OVERRIDE_GFX_VERSION', $null, 'Machine')`. + For fix-9, run it before applying the fix; if a value is already set, do + not apply it, and tell the user to change that value by hand if they + want a different GPU, as the CLI itself says when its read works. + 4. **Verify.** Have the user run the `verify` command from the diagnosis. Use `rocm examine` (or `rocm examine --json`) when you only need the host state @@ -321,7 +368,8 @@ start of the reply, before relaying a multi-step plan. - Never invent a fix. If `rocm diagnose` returns no match, route upstream. - Never run a mutating fix without the user's explicit OK; prefer `--dry-run` - first. New failure modes are added to the CLI catalog, not improvised here. + first (on Windows that dry run can itself exit 4 when its read fails; see + step 3). New failure modes are added to the CLI catalog, not improvised here. Read [reference.md](reference.md) when `rocm fix` exits non-zero (its exit codes are listed there), before telling the user which platforms a fix id applies to, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json index 9c0ed11c4b..105fa7e6e3 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json index 6b3f182c39..11a2e84b07 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json index 5592989a87..53a62d268e 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json index f98514ddff..0ce2ef0f49 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json index b45069b6ca..4e8ee964fb 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json index eb3b977605..476d2694c5 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json index d385254b59..ac06cbab70 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json @@ -5,7 +5,7 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran exited non-zero or could not be started, a file it wrote could not be written, or a read of state it needed before planning did not complete (on Windows, the current User value that fix-2, fix-6, and fix-9 with a device index read first, and for fix-2 the Machine value too, even under `--dry-run`). The change that step was to make was not made. If a command instead did not finish (it timed out or was stopped), the change may or may not have been made: stderr says how to check it and which `rocm fix` to re-run.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/reference.md b/skills/rocm-doctor/reference.md index 8329422d10..7e2dbf43f8 100644 --- a/skills/rocm-doctor/reference.md +++ b/skills/rocm-doctor/reference.md @@ -101,11 +101,11 @@ id is refused, and the refusal exits 1 (see the table). Exit codes: | code | meaning | | --- | --- | -| 0 | applied / dry-run / print-only plan / list | +| 0 | applied / dry-run (except the Windows read failure under 4) / print-only plan / list | | 1 | internal error; also the refusal of `--json` together with a fix id | | 2 | usage error (incl. unknown fix-id) | | 3 | not applicable on this host (OS mismatch, negative `--device-index`) — nothing changed | -| 4 | attempted but the command failed | +| 4 | attempted but the command failed or timed out; after a timeout the change may be partial — the message names how to check it and what to re-run (an older CLI instead prints `exited 124` or `exited -1` with `NOT changed`; see step 3 in [SKILL.md](SKILL.md)). Also, on Windows only, `fix-2-unset-override`, `fix-6-path` or `fix-9-igpu-dgpu --device-index N` when it could not read the current value of the variable it changes, `--dry-run` included — nothing changed and no plan shown; the catalog entry printed first is the fix's generic description, not a preview of the change on this machine | | 5 | user declined at the prompt | Three ids are ever auto-applicable — never four, and `fix-9-igpu-dgpu` is not @@ -134,8 +134,10 @@ a promise that the runner mutates anything, and — per the per-host list above not the same claim on Windows or WSL2 for the one entry whose behaviour depends on more than the host alone. -The ones that do mutate print the exact command, honor `--dry-run`, refuse on a -non-interactive shell without `--yes`, and confirm first. **Two exceptions:** +The ones that do mutate print the exact command, honor `--dry-run` (on +Windows a dry run whose read fails exits 4 with no plan; see 4 in the exit-code +table), refuse on a non-interactive shell without `--yes`, and confirm first. +**Two exceptions:** - **`fix-2-unset-override` mutates on Windows only.** Its Linux runner reports where the override is set and which rc files carry it, then stops — it will diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 3efc130f5d..2c61136e44 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -74,6 +74,10 @@ url = "2" vt100 = "0.18" [dev-dependencies] +# `ENGINE_PROTOCOL_VERSION`, so the engine protocol scenarios name this +# release's version from the constant the binaries stamp rather than a copy of +# it. A dev edge, which `xtask check-crate-edges` exempts. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } # Reads a printed command line back the way a POSIX shell splits words, so a # scenario can assert what a `tool_call:` line denotes rather than one spelling # of it. @@ -90,10 +94,11 @@ shlex.workspace = true # itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping # it here needs no allowlist entry. # -# For the same reason the library's `device_memory::bounded_stdout` cannot call -# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll, -# kill and reap steps on `std` alone, deliberately, and a normal dependency -# would be a new edge to review. A change to either one should look at the other. +# For the same reason the library's `device_memory::bounded_stdout` and +# `capability::output_within` cannot call `rocm_core::process::run_with_timeout`: +# each repeats that helper's spawn, poll, kill and reap steps on `std` alone, +# deliberately, and a normal dependency would be a new edge to review. A change +# to any one of these copies should look at the others. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d005021bc3..c68381db73 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -64,7 +64,7 @@ runs the suite. Extra arguments after `--` are forwarded to the cucumber CLI. # All features: cargo xtask e2e -# Filter by scenario name: +# Filter by scenario name (bypasses capability gating; see below): cargo xtask e2e -- -n "model name" # Stop on first failure: @@ -101,6 +101,57 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e | `ROCM_CLI_CONFIG_DIR` | (temp dir) | Isolated config directory per scenario | | `ROCM_CLI_DATA_DIR` | (temp dir) | Isolated data directory per scenario | | `ROCM_CLI_CACHE_DIR` | (temp dir) | Isolated cache directory per scenario | +| `E2E_PROBE_TIMEOUT_SECS` | 120 | How long each capability-probe command (`rocm examine`, `rocm engines list`) may run before it is killed and the run is refused: whole seconds ≥ 1; anything else uses 120. The post-run version probe (`collect_versions`) uses the same deadline but fails quietly, only blanking the version columns, and does not refuse the run | +| `E2E_EXPECT_GPU` | unset (undeclared) | Whether the lane has a GPU: `1` = GPU expected, `0` = deliberately none, unset or empty = no claim; any other value fails the run. Set on every self-hosted CI lane; see "Lane verdict" below | + +### Lane verdict + +Scenarios that resolve to skip never reach `report.json`, so scenario results +alone cannot tell a lane that tested its GPU from one on which every GPU +scenario was skipped. The harness checks the run itself, before and after the +scenarios: + +- **Before any scenario runs**, it fails when the capability probe could not + read the binary under test, naming the command and saying why: + `rocm examine` or `rocm engines list` could not be started (with the error), + or could not be waited on, exited non-zero (with the exit + status and the tail of its stderr), printed nothing, was still running after + `E2E_PROBE_TIMEOUT_SECS` (default 120; it is killed, and the message gives + the deadline and the stderr tail so far), or exited but left its output + open, held by a process it started (with the stderr tail so far). On a lane + that sets `E2E_EXPECT_GPU=1` it also fails when the probe found no usable AMD + GPU, naming the detected gfx target, WSL flag, driver status and visibility + mask it decided from. A run refused here writes no `platform.json`; its + `report.json` is already there, empty, because `run_suite` in + [`tests/e2e.rs`](tests/e2e.rs) creates it before calling + `lane_verdict::preflight`, and on every self-hosted lane that ran its suite + the results upload step is gated on `always()` (`e2e-gpu`'s also skips a + run that found itself stale, which never reaches the suite), so the + consolidated report shows the lane as EMPTY. +- **After the run**, it fails when no scenario executed (for example a `-n` + filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no + `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU scenarios + the run's own configuration leaves out (`@nightly`, `@merge-queue`, or + anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved + but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and + no GPU scenario selected fails by design. The reason is also recorded as + `lane_failure` in `platform.json`, so the consolidated report shows the lane + as FAIL, even when every scenario it resolved was a skip, and gives the reason + under "Needs attention: platforms" (in both the step summary and the HTML + report) and in the lane's section of the HTML report. + +`E2E_EXPECT_GPU=0` is for a lane that deliberately has no usable GPU, such as +the WSL lanes: the harness prints a note that `@requires-gpu` scenarios are not +applicable, and a note that the declaration is stale if the probe finds a GPU +anyway; neither fails the run. Both checks are in +[`src/lane_verdict.rs`](src/lane_verdict.rs). + +A cucumber `-n`/`--name` or `--tags` filter on the command line replaces the +harness's own scenario filter, so the scenarios it selects run without the +capability gating, the opt-in sets and the expectation matrix described below, +and the after-run GPU check does not apply. Use it to run a known scenario on a +host that can satisfy it; use `E2E_ONLY_LIFECYCLE` with `E2E_ALSO_RUN_IDS` to +select scenarios with the gating kept. ## Tags and per-scenario expectations @@ -137,7 +188,7 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:-` | Stable scenario id, prefixed with its feature's key. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. | +| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. On a lane that sets `E2E_EXPECT_GPU=1` a host with none fails the run instead; see "Lane verdict" above. | | `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. | | `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | | `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | @@ -145,10 +196,10 @@ Scenarios carry stable-id and capability tags: | `@requires-case-sensitive-fs` | Premise is two files whose names differ only in letter case. Resolves to **skip** where the scenarios' temp root folds case. The capability probe's `case_sensitive_fs` writes two such names in its own temp dir, which follows `$TMPDIR` like every scenario's isolated root, and checks they stayed two files. `@requires-os:linux` cannot express this: `$TMPDIR` on Linux, WSL2 included, can sit on a casefolded directory or a mounted Windows drive. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | -| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch that runs out of GPU memory so the OOM guidance can be verified without a GPU. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | +| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch so `rocm serve`'s summary can be verified without a GPU: one that runs out of GPU memory (the OOM guidance), or, in its `ready` mode, one that becomes ready and then fails its smoke test. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | -| `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios and the native-Windows vLLM refusal scenarios serve-33/34/35. | +| `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios, the native-Windows vLLM refusal scenarios serve-33/34/35, and the side-by-side runtime scenarios runtime-lifecycle-19/20/21. | Known bugs are **not** tagged in the `.feature` files — they live in `expectations.toml`, keyed by `@id`, each with a `when = { ... }` condition (e.g. @@ -194,18 +245,16 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow covers a strict superset of the hardware in the table -above, as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, -`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-w7900`, `e2e-gpu-nightly-strix`, +The nightly workflow runs the MI300X and the Strix Halo hardware from the +table above as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform -grid. `e2e-gpu-nightly-rad3` and `e2e-gpu-nightly-mi350p` have no per-PR -counterpart — both were demoted from per-PR to nightly-only per ROCMAI-125 — -so nightly is where their regression coverage now lives. The shared -large-model scenario follows each lane's serve engine: it serves -`Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), and the -`unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified on -Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). +grid. R9700, MI350P and W7900 no longer have a self-hosted nightly lane in +`nightly.yml`: they run on the queue ([nightly-on-the-queue.md](../../docs/nightly-on-the-queue.md)), +and the nightly report adds the queue's lanes to that grid. The shared large-model scenario follows each lane's serve engine: it +serves `Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), +and the `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified +on Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index feddea5300..2274bae5b0 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -243,10 +243,10 @@ reason = "Lemonade managed serve never exposes the model: observed backend argum serve_timeout_secs = 90 flaky = true -# --- Lemonade 11.5.1 cannot align its llama.cpp backend to a ROCm 7.14.x SDK --- +# --- Lemonade 11.5.1 could not align its llama.cpp backend to a ROCm 7.14.x SDK --- # `rocm engines install lemonade` tries to align Lemonade's `llamacpp:rocm` -# backend with the active ROCm SDK. Lemonade 11.5.1, the pin in -# `runtime-deps.toml`, downloads the TheRock tarball it pairs with the backend +# backend with the active ROCm SDK. Lemonade 11.5.1, which `runtime-deps.toml` +# pinned when these rows were written, downloads the TheRock tarball it pairs with the backend # only from https://repo.amd.com/rocm/tarball/, a path the 7.14.x tarballs are not # published to (AMD's 7.14 install guide uses /rocm/tarball-multi-arch/), so the # download answers HTTP 403 for `therock-dist---7.14.1.tar.gz`. The @@ -269,22 +269,21 @@ flaky = true # `serve_timeout_secs`: serve-06's own `@serve-timeout:2400` tag takes precedence # over a row, so a shorter wait here would have no effect, and the failing # serve-06 still holds the Windows lane for the full 2400s wait. +# +# The pin is now Lemonade 2026.44.1, which fetches TheRock from AMD's whl-next +# wheel index or its core tarball location, and whose pinned llama.cpp build +# (b11580) ships a ROCm 7.14 archive. So both rows may now XPASS. That has not +# been observed yet: both scenarios are `@nightly`, which pull-request lanes do +# not run. A stale XPASS on the first Windows nightly run after the 2026.44.1 pin +# lands means the cause is gone, and the row is deleted then, in a follow-up by +# the author of the 2026.44.1 pin. ROCMAI-899 is already closed; the row stays +# until that nightly shows the XPASS. [["runtime-lemonade-backend-alignment-reported"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-899" -reason = "Lemonade 11.5.1 fetches TheRock only from repo.amd.com/rocm/tarball/, where 7.14.x is not published (HTTP 403), so its backend cannot be aligned to a 7.14.x SDK and the install reverts to the pinned 7.13 backend." +reason = "Observed with Lemonade 11.5.1, which fetched TheRock only from repo.amd.com/rocm/tarball/, where 7.14.x is not published (HTTP 403), so its backend could not be aligned to a 7.14.x SDK and the install reverted to the pinned 7.13 backend. Not yet re-checked with the 2026.44.1 pin, which fetches TheRock elsewhere; delete this row if the Windows nightly lane reports it as a stale XPASS." [["serve-large-model-inference"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" -reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." - -# Scoped to native Linux Strix Halo, the only place it has been seen (evidence is -# in the tracking issue); `is_wsl = false` keeps WSL2, which also reports -# `os = "linux"` on gfx1151, out of it, so a WSL host that gains Docker and passes -# the scenario is not an XPASS. A host without Docker skips the scenario before -# this row applies. -[["serve-aim-lifecycle-parity"]] -when = { os = "linux", is_wsl = false, therock_family = "gfx1151*", has_amd_gpu = true } -bug = "rocm-cli#1058" -reason = "Stopping a running AIM service on Strix Halo Ubuntu reports `Local server stop not confirmed` (exit 1); the serve itself works." +reason = "The Windows large-model serve crashed in the SDK's Tensile library on the first inference request while Lemonade 11.5.1 was pinned: its ROCm 7.13 backend ran against the 7.14.1 SDK because 11.5.1 could not align it (suspected cause, not reproduced off Windows). Not yet re-checked with the 2026.44.1 pin; delete this row if the Windows nightly lane reports it as a stale XPASS." diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index f882460da1..16a360d0ba 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -185,3 +185,39 @@ Feature: Chat and endpoint detection When the user asks the CLI to plan a vague request Then the planning assistant was asked exactly once And the CLI plans the request itself and says the planning assistant's plan "must request managed serving with --managed" + + # The approval card is the user's only say over what an assistant tool call + # does, so the card and the command its approval runs must be the same one. + # A tool call carrying a second set of arguments nested inside the first is + # ambiguous about which it means: it is refused before any card is shown, + # and the model is told why, instead of showing one command and running the + # other. + @id:chat-ambiguous-tool-call-refused-before-approval @requires-os:linux + Scenario: chat-17 - A tool call carrying two sets of arguments is refused before any approval prompt + Given a running managed model answers a request to start a server with a tool call carrying two sets of arguments + When the user opens interactive chat + Then the local endpoint is shown for confirmation + When the user accepts the local endpoint + And the user asks the managed model to start a model server + Then the model is told the tool call was refused + And no tool approval prompt is displayed + When the user quits interactive chat + Then interactive chat exits successfully + + # `rocm mcp-call` is how a tool-using client runs a ROCm tool directly. A + # client may hand over the tool-call request it was given whole, with the + # tool's name beside its arguments; the CLI then runs the tool that request + # names, with the arguments inside it. + @id:chat-mcp-call-runs-a-whole-tool-call-request + Scenario: chat-18 - A whole tool-call request handed to the CLI runs the tool it names + When a tool-using client hands the CLI a whole tool-call request to check a path that exists + Then the tool reports that path as an existing directory + + # The same refusal as chat-17, on the direct path: a request holding a + # second set of arguments nested beside the first is refused rather than + # run with either set. + @id:chat-mcp-call-refuses-two-sets-of-arguments + Scenario: chat-19 - A direct tool request carrying two sets of arguments is refused and runs nothing + When a tool-using client asks the CLI to check a path with a second set of arguments nested inside the first + Then the CLI refuses the request for its nested arguments + And no tool result is printed diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 776e69a01e..94962ec519 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -215,6 +215,11 @@ Feature: Diagnosing failures and listing fixes # `fix-4-render-group`'s `applies_on` does not include `wsl`, so on a WSL2 # host the CLI refuses it as the wrong platform before ever invoking the # (faked) `usermod` — there is no command-failure branch to reach there. + # + # The Then step also holds the claim against the state: the stand-in records + # that it was asked to add the user to the groups, and the success line is + # absent, so "NOT changed" is said about a change that was attempted and + # refused, not about one that never ran or quietly went through. @id:diagnose-fix-command-failure-reported-on-stderr @requires-os:linux @requires-bare-metal Scenario: diagnose-15 - A fix whose helper command fails explains why, on stderr, with exit code 4 Given a user who has approved a fix whose helper command will fail @@ -610,7 +615,8 @@ Feature: Diagnosing failures and listing fixes # `ps_env_scope`'s doc comment names -- "PowerShell missing") rather than # faking the read failing, so the abort this proves is a real one, not a # simulated one, and it never touches the real User PATH at all, so no - # capture/restore is needed here. + # capture/restore is needed here. diagnose-69 and diagnose-70 hold fix-2 and + # fix-9 to the same rule. @id:diagnose-fix-path-aborts-on-unreadable-user-path @requires-os:windows Scenario: diagnose-39 - A fix-6-path run that cannot read the real User PATH aborts instead of wiping it Given a Windows host with a HIP SDK install and no way to resolve PowerShell @@ -909,7 +915,8 @@ Feature: Diagnosing failures and listing fixes # paste is diagnosed (not investigated further; the console is started with # cursor inheritance, which waits on a reply the screen emulator does not # send). That leaves the Windows half of the hint (Ctrl-Z then Enter) without - # an e2e check; docs/manual-testing.md section 13 covers it by hand. + # an e2e check; docs/manual-testing.md's "Pasting An Error Into `rocm diagnose`" + # section covers it by hand. @id:diagnose-a-paste-at-the-terminal-is-explained-and-diagnosed @requires-os:linux Scenario: diagnose-60 - Pasting the error at the terminal is explained, and the paste is diagnosed when the user ends it Given a pasted error containing quotes, a command substitution and a leading minus sign @@ -996,3 +1003,54 @@ Feature: Diagnosing failures and listing fixes Scenario: diagnose-67 - Diagnosing a failed load of the ROCm runtime library names the release the framework and the system must share When the user diagnoses this machine with a ROCm runtime library load failure Then the wheel mismatch cause and its plan name the X.Y release, not the major version + + # The sibling of diagnose-15 the reported bug was about: the helper command + # never finishes. The CLI stops it at the fix's one-minute bound, but it can + # only stop the command it started (`sudo`), so the change may still have + # been made. Before ROCMAI-345 this was reported as "usermod exited 124; + # group membership NOT changed." -- naming no timeout and claiming a state + # nobody had checked. The stand-in records its arguments and then sleeps far + # past the bound, so the scenario waits out the real minute: there is no + # test-only way to shorten it, on purpose, so the bound under test is the one + # users get. Same lane as diagnose-15, for the same reasons. + @id:diagnose-fix-command-timeout-says-change-may-be-partial @requires-os:linux @requires-bare-metal + Scenario: diagnose-68 - A fix whose helper command never finishes says the change may be partial and how to check it + Given a user who has approved a fix whose helper command never finishes + When the user asks the CLI to apply the approved fix + Then the CLI reports on stderr, with exit code 4, that the helper was stopped after a minute, how to check the change and which fix to re-run + And the CLI claims neither that the change was made nor that nothing changed + And the helper was asked to add the user to the render and video groups + + # A User-scope read that cannot complete used to be taken as "not set": with + # nothing in the Machine scope either, fix-2 printed that no persistent + # override was found and the user was done, and exited 0, on a machine that + # may still carry it. Same real "PowerShell missing" cause as diagnose-39. + # Applied for real: should the guard regress, both reads would read as unset + # and fix-2 would find nothing to clear, so the runner's User environment is + # never written. + @id:diagnose-fix-override-aborts-on-unreadable-user-scope @requires-os:windows + Scenario: diagnose-69 - Clearing the GPU override stops when the saved value cannot be read, instead of calling it gone + Given a Windows host where PowerShell cannot be found + And a user who has chosen the fix for the GPU override + When the user asks the CLI to apply the approved fix + Then the CLI stops with exit code 4, saying it could not read the saved override and that nothing was changed + And the CLI does not report the override as already gone + + # The same failed read made fix-9 skip its "already set" guard and overwrite + # a User-scope device setting the user may have chosen on purpose. Previewed + # rather than applied: the empty PATH hides PowerShell but not setx.exe, so + # an applied fix-9 whose guard regressed would set HIP_VISIBLE_DEVICES on the + # runner itself, with nothing to restore it. The preview reads the setting + # before planning, the same way the apply does, and a preview never runs + # setx. So this holds the stop and what the preview prints: the catalog entry + # still appears, the change it would make does not. That an applied fix-9 + # writes nothing after a failed read is held by the unit tests in + # fix/runners.rs (`pin_device_read_timeout_stops_before_setx`, + # `a_read_that_exits_non_zero_stops_every_windows_fix_without_writing`). + @id:diagnose-fix-igpu-aborts-on-unreadable-user-scope @requires-os:windows + Scenario: diagnose-70 - A dry run of the discrete-GPU pin that cannot read the saved device setting stops with exit code 4 and previews no change + Given a Windows host where PowerShell cannot be found + And a user who has chosen the fix that hides the integrated GPU + When the user previews that fix for the second GPU + Then the CLI stops with exit code 4, saying it could not read the saved device setting and that nothing was changed + And the fix's catalog entry is printed, but not the device setting it would save diff --git a/tests/e2e-cucumber/features/engine_ipc.feature b/tests/e2e-cucumber/features/engine_ipc.feature index 96eb773636..a2ce1f5057 100644 --- a/tests/e2e-cucumber/features/engine_ipc.feature +++ b/tests/e2e-cucumber/features/engine_ipc.feature @@ -1,25 +1,41 @@ -Feature: How the daemon reads an engine's answer +Feature: How the daemon and an engine read each other over stdio # `rocmd` asks an engine to do things (here: stop a server) by running # `rocm __engine-stdio ` and reading what the child prints. The # standalone `rocmd` binary has no such verb of its own and runs the `rocm` - # beside it, which is what makes these scenarios stageable from outside: each - # one copies the real `rocmd` into a scratch directory and puts a stand-in + # beside it, which is what makes engine-ipc-01 to -07 stageable from outside: + # each copies the real `rocmd` into a scratch directory and puts a stand-in # `rocm` next to it, so the daemon's own tool reaches a child whose answer the # scenario controls. The stand-in records the argv and the request it was # given and prints what the scenario says. # - # service-stop-02 covers the engine that answers cleanly; the two below are the - # other ways an engine child ends. In each, the engine's message in the tool's + # service-stop-02 covers the engine that answers cleanly; engine-ipc-01 and -02 + # are the other ways an engine child ends; -03 and -04 are answers the daemon + # refuses (written in an engine protocol it cannot use, or not alone on + # stdout), and -05 to -07 answers it reads (in its own protocol, in a newer + # one that only adds to it, and in none, as from an engine older than the + # field). In each, the engine's message in the tool's # report is checked together with what the stop did to the server: its process, # its record and its endpoint key. The server is a `vllm` one, for which the # engine's `Stop` is advisory (the recorded process decides whether the stop # is confirmed), so the stop going through is part of what is shown, not a # side effect. # - # Linux-only, for the same reason as service-stop-02: the recorded process - # carries the start-time token `/proc` gives it, and the stand-in is a shell - # script. No GPU and no real engine are involved, so these run on the mock lane. + # engine-ipc-08 to -12 are the other direction: the real built-in `vllm` + # engine, run as `rocm __engine-stdio vllm` the way the daemon runs it, reads a + # request. -08 is a stop it cannot serve, for a server whose process the + # engine itself recorded; -09 and -10 are requests it refuses for their + # protocol before it decodes their method (one it does not have) or reads a + # version that is not even text; -11 and -12 are a method it does not have in + # its own protocol, and input that is not a request at all. Each is answered + # on stdout, with a clean exit. + # + # engine-ipc-01 to -08 and -10 are Linux-only, for the same reason as + # service-stop-02: the recorded process carries the start-time token `/proc` + # gives it, and in -01 to -07 the stand-in is a shell script. -09, -11 and + # -12 use neither, but have only been run on Linux, so they keep the tag until + # they are run on Windows. No GPU and no real engine server are involved, so + # these run on the mock lane. @id:engine-ipc-reads-an-answer-printed-before-a-failing-exit @requires-os:linux @@ -45,3 +61,101 @@ Feature: How the daemon reads an engine's answer And the recorded process is no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone + + @id:engine-ipc-refuses-an-answer-in-a-protocol-it-cannot-use + @requires-os:linux + Scenario: engine-ipc-03 - An engine's answer in an engine protocol this release cannot use is refused, and the stop still completes + Given a managed server whose recorded process is running + And the engine answers the stop request in an engine protocol this release cannot use + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer refused, naming the engine's protocol and this release's + And the tool still reports the server stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-refuses-an-answer-with-other-output-on-stdout + @requires-os:linux + Scenario: engine-ipc-04 - An engine's answer with a log line before it on stdout is refused, and the stop still completes + Given a managed server whose recorded process is running + And the engine writes a log line on stdout before its answer to the stop request + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer refused for the log line the engine wrote before it + And the tool still reports the server stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reads-an-answer-in-its-own-protocol + @requires-os:linux + Scenario: engine-ipc-05 - An engine that answers in this release's engine protocol is read as having answered + Given a managed server whose recorded process is running + And the engine answers the stop request in this release's engine protocol + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reads-an-answer-in-a-newer-compatible-protocol + @requires-os:linux + Scenario: engine-ipc-06 - An engine that answers in a newer engine protocol that only adds to this release's is read as having answered + Given a managed server whose recorded process is running + And the engine answers the stop request in a newer engine protocol that only adds to this release's + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reads-an-answer-that-names-no-protocol + @requires-os:linux + Scenario: engine-ipc-07 - An engine that names no engine protocol, as one from before the field, is read as having answered + Given a managed server whose recorded process is running + And the engine answers the stop request in no named engine protocol + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-engine-refuses-a-request-in-a-protocol-it-cannot-serve + @requires-os:linux + Scenario: engine-ipc-08 - An engine asked to stop a server in an engine protocol it cannot serve refuses, and leaves the server running + Given an engine server whose process the vllm engine recorded is running + When the vllm engine is asked over stdio to stop that server, in an engine protocol it cannot serve + Then the engine answers that it cannot serve that engine protocol, naming its own and the request's + And the engine server's process is still running + And the vllm engine's record of that server still names its process + + @id:engine-ipc-engine-refuses-an-unknown-method-in-a-protocol-it-cannot-serve + @requires-os:linux + Scenario: engine-ipc-09 - An engine sent a method it does not have, in an engine protocol it cannot serve, refuses it for the protocol + When the vllm engine is sent over stdio a request for a method it does not have, in an engine protocol it cannot serve + Then the engine answers that it cannot serve that engine protocol, naming its own and the request's + + @id:engine-ipc-engine-refuses-a-version-that-is-not-text + @requires-os:linux + Scenario: engine-ipc-10 - An engine asked to stop a server with a protocol version that is not text refuses, and leaves the server running + Given an engine server whose process the vllm engine recorded is running + When the vllm engine is sent over stdio a stop request for that server whose protocol version is not text + Then the engine answers that it cannot read the request's protocol version + And the engine server's process is still running + And the vllm engine's record of that server still names its process + + @id:engine-ipc-engine-answers-an-unknown-method + @requires-os:linux + Scenario: engine-ipc-11 - An engine sent a method it does not have, in its own engine protocol, says it has no such method + When the vllm engine is sent over stdio a request for a method it does not have, in its own engine protocol + Then the engine answers that it has no such method + + @id:engine-ipc-engine-answers-input-that-is-not-a-request + @requires-os:linux + Scenario: engine-ipc-12 - An engine sent something that is not a request says so in an answer + When the vllm engine is sent over stdio something that is not a request + Then the engine answers that what it was sent is not a request diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 9e0dc5af00..a5a7d1c52b 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -540,7 +540,7 @@ Feature: GPU detection and system inspection And the inspection's driver detail says the library is missing from the linker cache # A TheRock tarball keeps librocdxg.so beside libhsa-runtime64, which opens it by - # name and carries `RUNPATH $ORIGIN`, so the loader finds it from the runtime's own + # name and searches its own `$ORIGIN` directory, so the loader finds it from the runtime's own # directory: no linker-cache entry and no LD_LIBRARY_PATH. ROCM-30386 reported such # a machine as `wsl_rocdxg_missing` while its `rocminfo` enumerated the GPU. The # control is `examine-33`, the same machine with the library alone. @@ -732,3 +732,64 @@ Feature: GPU detection and system inspection When the user inspects that machine in machine-readable form and the time it takes is measured Then the inspection finishes long before the background process does And the inspection lists the GPU that lspci printed + + # A ROCm wheel ships `librocdxg.so.1` beside `libhsa-runtime64`, but the HSA + # runtime opens `librocdxg.so`, a name the wheel does not ship, so on WSL2 it + # loads the host's librocdxg and never the copy it was built with. A 10.2 or + # newer nightly runtime then fails at first GPU use against a host + # `rocdxg-roct` 1.2.2 (`undefined symbol: hsaKmtSetPersistingCacheSize`), while + # `driver_status` still reads `wsl_rocdxg_ready`. The inspection lists such a + # copy; the control is the same runtime with `librocdxg.so` beside it. The + # install side of the same change (`rocm install sdk` linking the name) needs a + # real wheel download and is covered by unit tests only. + @id:examine-lists-a-runtime-librocdxg-its-runtime-does-not-load @requires-os:linux + Scenario: examine-49 - The inspection lists a managed runtime's librocdxg that its own runtime does not load + Given a WSL machine with a managed runtime that ships only a versioned librocdxg + When the user inspects that machine in machine-readable form + Then the inspection reports the ROCDXG driver status "wsl_rocdxg_ready" + And the inspection lists that runtime's librocdxg as one its own runtime does not load + + @id:examine-lists-no-runtime-librocdxg-when-the-runtime-loads-its-own @requires-os:linux + Scenario: examine-50 - The inspection lists nothing when the runtime has librocdxg.so beside its own copy + Given a WSL machine with a managed runtime that has librocdxg.so beside its own copy + When the user inspects that machine in machine-readable form + Then the inspection lists no runtime librocdxg that its own runtime does not load + + # The advice names a command: it has to exist and has to clear the finding. + @id:examine-runtime-librocdxg-advice-clears-the-finding @requires-os:linux + Scenario: examine-51 - Running the command the inspection advises clears the runtime librocdxg finding + Given a WSL machine with a managed runtime that ships only a versioned librocdxg + When the user runs the command the inspection advises for that runtime's librocdxg and inspects again + Then the inspection lists no runtime librocdxg that its own runtime does not load + + # The finding is advice about the WSL loader path; a native host with the same + # runtime gets none. + @id:examine-lists-no-runtime-librocdxg-off-wsl @requires-os:linux + Scenario: examine-52 - The inspection lists no runtime librocdxg on a machine that is not WSL + Given a machine that is not WSL with a managed runtime that ships only a versioned librocdxg + When the user inspects that machine in machine-readable form + Then the inspection lists no runtime librocdxg that its own runtime does not load + + # A broken `librocdxg.so` link is as bad as none: the loader's open() fails on + # it as on a missing file and the runtime loads the host's copy (observed with + # LD_DEBUG=libs on the 10.2 nightly, which still exited 139). The inspection + # lists it, and the advised command (which must replace a dangling link, so a + # plain `ln -s` is not enough) leaves the name reading as the runtime's own copy. + @id:examine-lists-a-dangling-runtime-librocdxg-link-and-the-advice-clears-it @requires-os:linux + Scenario: examine-53 - A dangling librocdxg.so link is listed and the advised command replaces it + Given a WSL machine with a managed runtime whose librocdxg.so is a link to a missing file + When the user inspects that machine in machine-readable form + Then the inspection lists that runtime's librocdxg as one its own runtime does not load + When the user runs the command the inspection advises for that runtime's librocdxg and inspects again + Then the inspection lists no runtime librocdxg that its own runtime does not load + + # `examine` reports the active runtime's channel, package source, source layout + # and exact GPU target in the same words and from the same manifest fields as + # the `provenance:` line `rocm runtimes list` prints for it + # (`runtime-lifecycle-22`). The planted manifest needs no GPU or download, so + # this runs on the mock lane. + @id:examine-reports-where-the-active-runtime-came-from + Scenario: examine-54 - Examining the system says where the active runtime came from + Given the active runtime is a nightly runtime installed for one exact GPU + When the user inspects the system + Then the runtime state names the channel, package source, source layout and exact GPU target that runtime was installed from diff --git a/tests/e2e-cucumber/features/install_lifecycle.feature b/tests/e2e-cucumber/features/install_lifecycle.feature index 025484b852..d7a9fd0707 100644 --- a/tests/e2e-cucumber/features/install_lifecycle.feature +++ b/tests/e2e-cucumber/features/install_lifecycle.feature @@ -468,3 +468,110 @@ Feature: Release install lifecycle When the user uninstalls from the installed binary Then the installed rocm and rocmd binaries are gone And the user's own fish completions are untouched + + # ── Default trust: the pinned release key with no signature requested ── + # + # Every other install here asks for a signature (ROCM_CLI_REQUIRE_SIGNATURE=1), + # so lifecycle-03 and -06 stay green under a regression that only bites when + # nobody asked. These two run as a user does, without asking: + # + # - lifecycle-36 installs a specific version with the variable set to 0, so + # only the pinned-key fallback is left to force the check (the release + # channel's own signature requirement does not apply). It catches an + # installer that verifies only when ROCM_CLI_REQUIRE_SIGNATURE is set, and + # one that verifies only when a signature is required, not merely when a + # key resolves. + # - lifecycle-37 is a release install with the variable unset and no .sig. It + # catches an installer that verifies only when ROCM_CLI_REQUIRE_SIGNATURE is + # set, and one that tolerates a missing .sig unless that variable is set, + # which lifecycle-36 (the .sig is there) and lifecycle-06 (the variable is + # set) both miss. It does not catch an installer that verifies only when a + # signature is required: the release channel's signature floor requires one, + # which masks that regression here. + # + # Neither can detect removal of the release channel's signature floor: while a + # release key is pinned, a resolved key already forces the check. Nor does + # lifecycle-36 check that the harness passes its channel through: if the step + # silently ran a release install instead, the floor would force the same + # rejection, and the second regression above would go unnoticed. + + @id:lifecycle-linux-default-rejects-untrusted-signer @lifecycle @requires-os:linux + Scenario: lifecycle-36 - Linux - installing a specific version with the signature requirement turned off still checks the pinned release key and rejects an untrusted signer + Given a freshly built release tree + And a generated signing keypair + And the user has turned the installer's signature requirement off + When the release is packaged and signed with the private key file + And a specific version of the signed bundle is installed with no public key supplied + Then the install fails reporting signature verification failed + And no binaries are activated in the target directory + + @id:lifecycle-linux-default-rejects-missing-signature @lifecycle @requires-os:linux + Scenario: lifecycle-37 - Linux - a release install that asks for nothing still rejects a bundle with no signature + Given a freshly built release tree + And a generated signing keypair + And the user has not asked the installer to require a signature + When the release is packaged and signed with the private key file + And the bundle signature sidecar is removed + And the signed bundle is installed with no public key supplied + Then the install fails reporting the required signature is missing + And no binaries are activated in the target directory + + # ── Download progress rendering ──────────────────────────────────────── + + @id:lifecycle-linux-progress-single-line @lifecycle @requires-os:linux + Scenario: lifecycle-38 - Linux - a download draws one progress bar that stays inside the terminal + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed through a pseudo-terminal narrower than COLUMNS + Then every download progress frame stays within the terminal + And the finished screen shows one completed bar per download + And the installed rocm binary is present + + @id:lifecycle-linux-progress-narrow-terminal @lifecycle @requires-os:linux + Scenario: lifecycle-39 - Linux - a terminal too narrow for a bar downloads quietly and says why + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed through a terminal too narrow for a progress bar + Then the installer reports that the terminal is too narrow for a progress bar + And the installed rocm binary is present + + @id:lifecycle-linux-progress-wget-fallback @lifecycle @requires-os:linux + Scenario: lifecycle-40 - Linux - the wget fallback keeps its progress bar inside the terminal + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed through a pseudo-terminal with curl unavailable + Then every download progress frame stays within the terminal + And the download prints no request logging around the bar + And the installed rocm binary is present + + @id:lifecycle-linux-progress-piped-into-shell @lifecycle @requires-os:linux + Scenario: lifecycle-41 - Linux - piping the installer into a shell still sizes the bar from the terminal + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed by piping the installer into a shell + Then every download progress frame stays within the terminal + And each completed bar spans the measured terminal width + And the installed rocm binary is present + + @id:lifecycle-linux-progress-unsized-terminal @lifecycle @requires-os:linux + Scenario: lifecycle-42 - Linux - a terminal reporting no size still gets a progress bar + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed where the terminal reports no size + Then the installer still draws a progress bar + And each completed bar spans the measured terminal width + And the installed rocm binary is present + + @id:lifecycle-linux-progress-zero-padded-columns @lifecycle @requires-os:linux + Scenario: lifecycle-43 - Linux - a zero-padded COLUMNS does not abort the install + Given a freshly built release tree + And a generated signing keypair + When the release is packaged and signed with the private key file + And the signed bundle is installed with a zero-padded COLUMNS and no way to measure + Then every download progress frame stays within the terminal + And the installed rocm binary is present diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index e55ad7fe16..8b79591a37 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -16,7 +16,8 @@ Feature: Lemonade installs on disk # None downloads anything or needs a GPU, so all run on the mock lane every PR. # lemonade-installs-01 and -02 need the `rocm/e2e-test-hooks` build # `cargo xtask e2e` makes, to report a nearly full drive without filling a - # real one. + # real one; lemonade-installs-07, -08 and -09 need it to report old host + # libraries on a host whose libraries are new. @id:lemonade-installs-refused-without-space Scenario: lemonade-installs-01 - A Lemonade install the drive cannot hold is refused before downloading @@ -62,3 +63,47 @@ Feature: Lemonade installs on disk Then the report does not say that copy is removed with its ROCm install When the user uninstalls the ROCm install holding Lemonade Then that Lemonade copy is still on disk + + # The pinned Linux Lemonade server links against a minimum glibc and OpenSSL. + # On an older host it used to exit with a loader error that reached the user + # only as "Lemonade server did not become ready"; the install now refuses + # first and names, for each library, what it needs and what the host has. The + # host reported is Ubuntu 20.04-like: glibc 2.31, and OpenSSL 1.1 only, so no + # libssl.so.3. Linux only: Windows has no such floor. + # + # The same refusal also guards starting the Lemonade server itself. No + # scenario reaches that one: `rocm serve` runs this install check first, a + # Linux serve with a packaged llama-server never starts the Lemonade server, + # and the mock lane has no GPU for serve to resolve. The unit tests in + # engines/lemonade/src/process.rs cover it. + @id:lemonade-installs-refused-on-too-old-host-libraries @requires-os:linux + Scenario: lemonade-installs-07 - A Lemonade install on a host with too old a glibc and no OpenSSL 3 is refused before downloading + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 + When the user installs the Lemonade engine + Then the Lemonade install is refused for the host's libraries before anything is downloaded + + # A host can provide OpenSSL 3 or glibc outside the loader cache the check + # reads (LD_LIBRARY_PATH, a Nix or Guix profile, a user prefix), and the + # refusal names an opt-out for it. The opt-out must actually get the install + # past the check: here it goes on until the drive, reported nearly full, stops + # it before any download, which proves the host-library check let it through + # without fetching anything. + @id:lemonade-installs-host-library-check-opt-out @requires-os:linux + Scenario: lemonade-installs-08 - Turning the host-library check off lets a Lemonade install go ahead with a warning + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 + And the user has turned off the Lemonade host-library check + And the drive for a new Lemonade install cannot hold even Lemonade itself + When the user installs the Lemonade engine + Then the user is warned that the host's libraries are too old and the install went ahead + And the Lemonade install is refused before anything is downloaded + + # A re-run of the install where the pinned Lemonade is already in place has + # nothing to download, so "Nothing was downloaded." would describe a branch the + # install was never on. The refusal says the installed copy was left as it is + # instead, and it was. + @id:lemonade-installs-refused-on-too-old-host-libraries-when-installed @requires-os:linux + Scenario: lemonade-installs-09 - Re-running a Lemonade install on a host too old for it leaves the installed copy as it is + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 + And the pinned Lemonade is already installed in the data folder + When the user installs the Lemonade engine + Then the Lemonade install is refused for the host's libraries and the installed copy is left as it is diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 3623c1e80a..e6fd7b0ebb 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -17,7 +17,7 @@ Feature: Model serving # This discriminates nothing yet: the scenario is an expected failure on every # host while its xfail row stands, and the vLLM run stops before printing a # plan, so the step panics there whatever the lemonade half did. It becomes - # regression coverage when that row is narrowed. serve-55 and the unit tests in + # regression coverage when that row is narrowed. serve-60 and the unit tests in # serve_cmd.rs are what hold the per-engine refs in the meantime. @id:serve-short-name-consistent-across-engines Scenario: serve-02 - Every engine expands a short name rather than serving it @@ -373,8 +373,8 @@ Feature: Model serving # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. # On a host without Docker both skip (the nightly job logs whether Docker is - # reachable). serve-27's known failure on Strix Halo Ubuntu is an xfail row in - # expectations.toml, tracked in rocm-cli#1058. + # reachable). serve-27's stop failure on Strix Halo Ubuntu (rocm-cli#1058) + # passed on the 2026-11-15 and 2026-11-16 nightlies, so its xfail row is gone. # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight @@ -940,3 +940,67 @@ Feature: Model serving And the response identifies the correct model When the user stops the scenario's managed Lemonade service Then the managed service is stopped and its endpoint is closed + + # A served endpoint's API key sits in `/services/.endpoint-key`, and + # nobody but the user who started the server may read it. Here: a loopback + # serve with `--require-api-key` (so no public port is opened on the GPU box), + # then the key file's mode is checked to be exactly 0600. The file is replaced + # by a freshly created owner-only one on every store, so a mode loosened in + # place is tightened again; that re-store case is covered by the unit tests in + # `endpoint_keys.rs` and `rocm-core`'s `atomic_write.rs`, since a stop deletes + # the key before a restart writes it back. The check does not depend on + # readiness: the serve waits a fixed 45 seconds for the engine + # (`READINESS_WAIT` in `service_launch/readiness.rs`) and then exits 0, + # reporting a server still loading as `starting`; the key is written before + # the engine starts, and the scenario's teardown stops the service. + # Linux-only: on Windows the key is protected by an owner-only DACL + # instead, but the Windows GPU lane serves with Lemonade, which refuses a + # key-protected bind there (`ensure_public_bind_engine_supported` in + # `service_launch/endpoint_auth.rs`), so the Windows half is covered by the + # `rocm-core` unit tests that read the DACL back on the Windows build job. + @id:serve-endpoint-key-readable-by-owner-only @requires-gpu @requires-os:linux + Scenario: serve-59 - Only the user who started a key-protected server can read its key + Given a managed runtime is active + When the user serves a model that requires an API key + Then the server's API key is kept where only that user can read it + + # A launch that became ready but whose first chat request fails must not be + # shown as a healthy deployment (#523). Same test-only hook as + # @id:serve-oom-launch-memory-guidance, in its + # `ready` mode, pointed at a mock engine that fails every chat request. The + # unit and provider tests prove the rendering and the failure classes; this + # proves `serve()` carries the smoke test's failure into the summary. + @id:serve-smoke-failure-summary @requires-no-gpu @requires-os:linux @requires-oom-fault-injection + Scenario: serve-60 - A launch whose first chat request fails is not presented as ready + Given a managed vLLM launch that becomes ready but whose engine fails its first chat request + When the user opens the interactive serve summary against that engine + Then the deployment summary says the smoke test failed and does not call the service ready + And the deployment summary gives the engine's own reason + And the engine received exactly one chat request + And the deployment summary names the logs command for that service + And the launch's service record is left as ready + And no background automation daemon was left behind + + # An API key on a vLLM endpoint (`--require-api-key`, a public bind, or + # `rocm remote serve`) has to cover the whole server, not only the OpenAI `/v1` + # API: vLLM's own key check guards a fixed set of path prefixes, while the + # server also answers inference and tokenizer routes outside them. rocm-cli + # starts vLLM with a middleware that demands the key on every route except + # `GET`/`HEAD /health` and CORS preflights. The middleware itself is + # unit-tested on every PR (`endpoint_auth` in the vLLM engine crate); this + # proves it is actually wired into a real vLLM server. A full extra vLLM + # serve, so `@nightly` rather than per-PR: it runs after merge in the nightly + # runs on Instinct GPUs, which serve with vLLM: the MI300X job in + # `nightly.yml` and the shared GPU queue's Instinct lanes (MI350P among them). + # Not `@merge-queue`: the merge queue runs no lane where vLLM is the engine + # (the MI300X lane does not run on `merge_group`), so under that tag it + # skipped everywhere. + @id:serve-vllm-key-guards-every-route @requires-gpu @requires-engine:vllm @nightly + Scenario: serve-61 - A key-protected vLLM endpoint refuses keyless requests outside the OpenAI API + Given a managed runtime is active + And a model is being served on GPU behind an API key + When a client calls the server's routes without the key + Then every route outside the OpenAI API refuses the call as unauthorized + And the health check answers without the key + When the client repeats those calls with the key + Then every one of those calls succeeds diff --git a/tests/e2e-cucumber/features/runtime_lifecycle.feature b/tests/e2e-cucumber/features/runtime_lifecycle.feature index 5f8da06c42..19bfe62a5b 100644 --- a/tests/e2e-cucumber/features/runtime_lifecycle.feature +++ b/tests/e2e-cucumber/features/runtime_lifecycle.feature @@ -262,3 +262,196 @@ Feature: Runtime lifecycle state machine And the server is still counted as left on the first runtime And the switch to the second runtime was kept And the server record still names the first runtime + + # ROCm 10 is published in its own layout, beside the canonical one the CLI + # installed from before. A runtime installed from it records that layout, and + # its key carries a fingerprint of the exact packages it was built from; a + # runtime installed before that layout existed has no top-level + # `source_layout_generation` or `devel` field: it records its layout and + # toolchain only inside `wheel_composition`. Both kinds stay installed side by + # side, so the CLI has to keep reading the older record as it was written while + # it lists, switches between and removes the newer ones. These scenarios plant + # three managed runtimes as an install would leave them — a registry entry, the + # same record inside the runtime's own folder, and stub files where the + # readiness check looks — without downloading anything. Every command runs + # with the startup update check disabled, since it would otherwise ask the + # package index about the planted runtimes. The Windows release lane in CI + # selects them by id, so they run on both supported platforms. + @id:runtime-lifecycle-side-by-side-across-packaging-change + Scenario: runtime-lifecycle-19 - Runtimes installed before and after the ROCm 10 layout are used side by side + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + When the user lists the installed runtimes + Then all three runtimes are listed as ready, each under its own key + When the user activates each of the three runtimes in turn + Then each one becomes the runtime in use in turn + And none of the three runtimes' install records was rewritten + + @id:runtime-lifecycle-side-by-side-uninstall-keeps-the-others + Scenario: runtime-lifecycle-20 - Uninstalling one side-by-side runtime leaves the others usable + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + And the nightly runtime is in use after the ROCm 10 release runtime + When the user uninstalls the ROCm 10 release runtime + Then that runtime's folder and record are gone + And the other two runtimes keep their folders and unchanged records + And the nightly runtime is still in use, with no rollback target left + When the user uninstalls the runtime in use + Then no runtime is in use + When the user activates the ROCm 7 runtime + Then the ROCm 7 runtime is the runtime in use + + # The ROCm 7 and ROCm 10 release runtimes share a name: the runtime id is + # derived from the channel and GPU family alone. Picking either silently could + # put the user on the wrong ROCm major version. + @id:runtime-lifecycle-side-by-side-shared-name-refused + Scenario: runtime-lifecycle-21 - A name shared by side-by-side runtimes is refused with each runtime named + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + When the user activates a runtime by the name the two release runtimes share + Then the CLI refuses and names both release runtimes + And no runtime is in use + + # The install report names a runtime's channel, package source, source layout + # and exact GPU target; afterwards only its manifest still knows them. The + # listing reads them back and says when one was never recorded rather than + # inferring it — the older runtime's family is itself an exact target, so a + # guess would look like a recorded value. An adopted runtime installed + # nothing, so it has no package source; its manifest holds only the marker + # adoption writes in place of one, in the on-disk form earlier releases wrote, + # and the listing must not show that marker as a source. The planted manifests + # need no SDK download or GPU, so this runs on the mock lane. + @id:runtime-lifecycle-list-names-runtime-provenance + Scenario: runtime-lifecycle-22 - Listing runtimes says which stream, source and GPU each runtime was installed from + Given a registered nightly runtime installed for one exact GPU + And a registered runtime installed before its source layout and GPU target were recorded + And a registered runtime adopted from an environment ROCm CLI did not install + When the user lists the registered runtimes + Then the nightly runtime's listing names its channel, package source, source layout and exact GPU target + And its version shows its build date + And the older runtime's listing says which of those facts were not recorded instead of guessing + And the adopted runtime's listing says it has no recorded package source + + # A script reads the runtime list without scraping the table. stdout must hold + # one JSON document and nothing else; diagnostics belong on stderr. The active + # and rollback flags are the facts the text marks with `*` and `-`, so the + # scenario also lists the same state as text and checks that both forms mark + # the same runtimes. + @id:runtime-lifecycle-list-json-marks-active-and-rollback + Scenario: runtime-lifecycle-23 - A script reads the runtime list as JSON + Given two registered runtimes with the second active after the first + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON lists both runtimes with their recorded values + And the JSON marks the second runtime active and the first as the rollback target + And the text listing marks the same runtimes as the JSON + + # The empty registry is where the text form prints placeholders (``) + # and a next-step hint; the JSON form has to stay a parseable document with + # nulls instead, not fall back to that text. + @id:runtime-lifecycle-list-json-empty-registry + Scenario: runtime-lifecycle-24 - Listing runtimes as JSON with none registered + Given no ROCm runtime is registered + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON lists no runtimes and no active runtime + + # A runtime whose folder was deleted outside the CLI stays registered, but + # activation would refuse it. A script has to see that from the document + # alone, without trying the activation. + @id:runtime-lifecycle-list-json-reports-unusable-runtime + Scenario: runtime-lifecycle-25 - The JSON runtime list reports a runtime whose folder is gone as unusable + Given two registered runtimes with the second active after the first + And the first runtime's folder has been deleted + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON reports the first runtime unusable because its folder is missing + And the JSON reports the second runtime ready + + # The active runtime's registry entry was removed by hand while the config + # still names it. The document has to say so, not just mark nothing active. + @id:runtime-lifecycle-list-json-reports-missing-active-manifest + Scenario: runtime-lifecycle-26 - The JSON runtime list says when the active runtime's registry entry is gone + Given two registered runtimes with the second active after the first + And the second runtime's registry entry has been deleted + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON reports that the active runtime key has no registered runtime + And the JSON marks no runtime active + + # `rocm config set-default-runtime` stores the id as typed and clears the + # active key, so the list decides the active runtime from the default id + # alone. An id that differs from a runtime's only in letter case still names + # that runtime, and the list must not also report its registry entry missing. + @id:runtime-lifecycle-list-default-id-in-other-letter-case + Scenario: runtime-lifecycle-27 - A default runtime set in other letter case is listed as active with no problem + Given two registered runtimes and none active + When the user sets the default runtime to the first runtime's id in other letter case + And the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON shows the default runtime as typed and no active runtime key + And the JSON marks only the first runtime active and reports no problem + And the text listing marks the first runtime active with no active_status line + + # A config edited by hand can name the active runtime in other letter case. + # The commands that act on the active runtime match its key ignoring case, so + # the list has to mark the runtime they act on, and `rocm examine` has to + # report that same runtime. The default runtime still names the second + # runtime, so a list that fell back to it would mark the wrong one. + @id:runtime-lifecycle-list-active-key-in-other-letter-case + Scenario: runtime-lifecycle-28 - An active runtime named in other letter case is listed as the one examine reports + Given two registered runtimes with the second active after the first + And the configuration names the first runtime as active in other letter case + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON marks only the first runtime active and reports no problem + And the text listing marks the first runtime active with no active_status line + And examine reports the first runtime as the active one + + # The configured active runtime is not registered, but the default runtime + # still names a registered one and the commands fall back to it. The list + # marks that runtime active, as `rocm examine` reports it, and still says the + # configured active runtime is missing, in both forms. + @id:runtime-lifecycle-list-dead-active-key-falls-back-to-default + Scenario: runtime-lifecycle-29 - A missing active runtime is reported while the default runtime the commands use is listed active + Given two registered runtimes with the second active after the first + And the configuration names an unregistered runtime as active + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON marks only the second runtime active + And the JSON reports that the active runtime key has no registered runtime + And the text listing marks the same runtimes as the JSON + And the text listing says the active runtime key has no registered runtime + And examine reports the second runtime as the active one + + # `--json` changes what a successful list prints, not how a failure is + # reported: errors are not JSON. A registry entry the CLI cannot read stops + # the list, so a script has to learn that from the exit code and an empty + # stdout rather than from a document that looks like a shorter list. + @id:runtime-lifecycle-list-json-failure-prints-nothing-on-stdout + Scenario: runtime-lifecycle-30 - A runtime list that cannot be read fails as JSON with nothing on stdout + Given two registered runtimes with the second active after the first + And the first runtime's registry entry cannot be read + When the user lists the registered runtimes as JSON + Then the list fails with nothing printed on stdout + And the error on stderr is plain text naming the unreadable registry entry + + # A script reads where each runtime came from out of the JSON listing instead + # of parsing the text's `provenance:` line. A fact the manifest never recorded + # has no value there, where the text prints `unrecorded`, so a script cannot + # take the placeholder for a recorded value. The same runtimes as + # `runtime-lifecycle-22` are planted, and the text listing of the same state + # is compared fact by fact, so the two forms are shown to agree. The startup + # update check is off: with wheel runtimes registered it looks for a Python + # and the network, and can print ahead of the document, which is not what + # this scenario is about. The planted manifests need no SDK download or GPU, + # so this runs on the mock lane. + @id:runtime-lifecycle-list-json-names-runtime-provenance + Scenario: runtime-lifecycle-31 - A script reads from the JSON runtime list which stream, source and GPU each runtime was installed from + Given a registered nightly runtime installed for one exact GPU + And a registered runtime installed before its source layout and GPU target were recorded + And a registered runtime adopted from an environment ROCm CLI did not install + And ROCm CLI does not check for runtime updates when it starts + When the user lists the registered runtimes as JSON + Then the output is a single JSON document and nothing else + And the JSON names the nightly runtime's channel, package source, source layout and exact GPU target + And the JSON gives no value for each fact the older runtime did not record instead of guessing + And the JSON gives no package source for the adopted runtime + And the text listing names the same provenance as the JSON for every runtime diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 24c61ea166..2114a7b355 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -330,26 +330,27 @@ Feature: Runtime configuration And the execution section says the consent came from the user's --yes # Lemonade's llama.cpp backend re-pins itself to match the ROCm SDK rocm-cli - # actually installed (Tier 1: point the pinned build at it; Tier 2: fall back - # to the newest build if the pin is too old to have shipped a matching - # ROCm-version asset; revert to the packaged default if neither verifies). - # The unit tests exercise that state machine directly against injected - # install/align steps, but nothing else asserts that `rocm engines install - # lemonade` actually surfaces the outcome to a real user -- this is the one - # part of that path with no other e2e coverage. - # - # `--reinstall` re-extracts the packaged embeddable, resetting - # `backend_versions.json` to its pinned defaults, so this fires - # deterministically even against a shared runtime tree where an earlier - # scenario already left Lemonade's backend aligned (a plain install would - # find nothing left to do and print no alignment line at all). Verified - # against real hardware (Strix Halo, gfx1151): a fresh SDK's version does not - # match Lemonade's packaged pin, Tier 1's install 404s (the pinned build - # predates a matching ROCm-version asset), and Tier 2's newest build - # succeeds -- producing exactly the line this scenario asserts. That held for - # an SDK whose TheRock tarball Lemonade could download; for a 7.14.x SDK Tier 2 - # fails too (the tarball is not published where Lemonade 11.5.1 looks), which is - # why `expectations.toml` carries a Windows row for this scenario (ROCMAI-899). + # actually installed (Tier 1: point the pinned build at it; Tier 2: the newest + # build that ships an archive for that ROCm version; revert to the packaged + # default if neither verifies). The unit tests exercise that state machine + # directly against injected install/align steps, but nothing else asserts that + # `rocm engines install lemonade` ends with a ROCm backend paired with the SDK + # on a real GPU -- this is the one part of that path with no other e2e + # coverage. + # + # Which route a lane takes depends on its SDK. An SDK older than Lemonade's + # packaged pin (ROCm 7.14 against 2026.44.1's 10.1.0) is aligned, and the CLI + # says so; an SDK that already is the packaged pin (ROCm 10.1) needs no + # alignment and gets none; a nightly SDK is never aligned and keeps the + # packaged pin. The Then accepts each route and fails a revert, or an + # alignment the install does not report: + # it reads the pin the installed runtime tree holds and compares it with the + # active runtime's version. `--reinstall` re-extracts the packaged embeddable, + # resetting `backend_versions.json` to its pinned defaults, so this does not + # depend on what an earlier scenario left in a shared runtime tree. + # `expectations.toml` carries a Windows xfail row for this scenario + # (ROCMAI-899), written for Lemonade 11.5.1, which could not fetch a 7.14.x + # TheRock tarball; a deterministic XPASS on that lane marks the row stale. # # `@requires-engine:lemonade` because vLLM shares the SDK's own runtime # environment and has no llama.cpp backend to align; `@nightly` for the same @@ -361,12 +362,12 @@ Feature: Runtime configuration # selected backend's llama-server, so the last Then can only pass if the # reinstall put it back. @id:runtime-lemonade-backend-alignment-reported @requires-gpu @requires-os:windows @requires-engine:lemonade @nightly - Scenario: runtime-16 - Reinstalling Lemonade reports whether its ROCm backend was aligned + Scenario: runtime-16 - Reinstalling Lemonade pairs its ROCm backend with the active SDK Given a managed runtime is active And the lemonade engine is installed And the selected Lemonade backend's llama-server binary has been deleted When the user reinstalls the lemonade engine - Then the CLI reports that Lemonade's ROCm backend was aligned to the active SDK + Then Lemonade's ROCm backend is paired with the active SDK And the selected Lemonade backend's llama-server binary is installed # `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` is the exit for a hand-edited diff --git a/tests/e2e-cucumber/features/service_record_cleanup.feature b/tests/e2e-cucumber/features/service_record_cleanup.feature index 73650eea28..4c19528302 100644 --- a/tests/e2e-cucumber/features/service_record_cleanup.feature +++ b/tests/e2e-cucumber/features/service_record_cleanup.feature @@ -6,11 +6,19 @@ Feature: Local server record cleanup # # A record owns FOUR files, and only two of them sit in the services folder: # the details file and the log live there, the engine state file lives under - # `/engines//state/`, and a public serve also leaves a 0600 - # endpoint key file. Deleting the two obvious ones — the documented manual + # `/engines//state/`, and a public serve also leaves an + # owner-only endpoint key file. Deleting the two obvious ones — the documented manual # workaround — is what orphans the rest, so these scenarios assert on all of # them rather than on the command's own summary line. # + # A save of the endpoint key that was killed part-way can also leave a second + # copy of the key beside it (`.endpoint-key.tmp---`, + # the shared atomic writer's staged file). It holds the key in plaintext, so + # removing a record deletes and counts it, and prune deletes it with a record + # it prunes and sweeps one whose record is gone. service-cleanup-01 plants one + # for the record it removes; -03 plants one for the record prune removes and + # one for a record deleted by hand. + # # A new feature file rather than appended scenarios: `feature_naming.rs` # requires per-file indexes to be sequential in declaration order, so an # appended scenario must claim its file's *next* index, and several open @@ -20,9 +28,12 @@ Feature: Local server record cleanup @id:service-cleanup-removes-every-file-a-record-owns Scenario: service-cleanup-01 - Removing a stopped record deletes every file it owns Given a local server record that is no longer running + And an interrupted save left a second copy of that server's API key When the user removes that local server record Then the CLI names the log file before deleting it And every file belonging to that record is gone + And the second copy of that server's API key is gone + And the CLI counts that copy among the files it removed And the shared launch lock is still there And the record no longer appears in the full list @@ -36,13 +47,18 @@ Feature: Local server record cleanup @id:service-cleanup-prunes-previews-then-sweeps-leftovers Scenario: service-cleanup-03 - Pruning previews first, then sweeps leftover engine state Given a local server record that is no longer running + And an interrupted save left a second copy of that server's API key And an engine state file whose local server record was deleted by hand + And an interrupted save left a copy of the API key of the record deleted by hand When the user previews a prune of every record that is not running Then the preview lists the record and the leftover file + And the preview lists the leftover copy of the API key And every file belonging to that record is still there When the user prunes every record that is not running Then every file belonging to that record is gone + And the second copy of that server's API key is gone And the leftover engine state file is gone + And the leftover copy of the API key is gone # The age rule only helps if the way past it is discoverable. A user who ran # `prune`, saw nothing happen and read the summary has to be able to act on @@ -84,7 +100,7 @@ Feature: Local server record cleanup And the record's other files are gone And the prune is still recorded in the audit log - # `rocm serve` writes the 0600 endpoint key file *before* it writes the + # `rocm serve` writes the owner-only endpoint key file *before* it writes the # record, so a server that is still coming up has a key with no record beside # it — exactly what the leftover sweep above looks for, and `--any-age` leaves # no age rule to hide behind. `serve` holds the shared launch lock across both diff --git a/tests/e2e-cucumber/features/vllm_tensorizer.feature b/tests/e2e-cucumber/features/vllm_tensorizer.feature new file mode 100644 index 0000000000..a404a52445 --- /dev/null +++ b/tests/e2e-cucumber/features/vllm_tensorizer.feature @@ -0,0 +1,142 @@ +Feature: vLLM's tensorizer dependency must import before an install is reported usable + + # vLLM's wheel pins tensorizer 2.10.1, which raises a KeyError at import under + # Python 3.14 (ROCm 10.1.0's vLLM is published for that Python only). Before the + # install checked for it, `rocm engines install vllm` reported its dependencies + # satisfied and the first `rocm serve` died in the engine core. + # + # The runtime's Python and `uv` are planted shell stand-ins: the Python cannot + # import tensorizer until the stand-in `uv` has installed the fixed release, and + # the stand-in `uv` answers `pip check` the way the real one does once a release + # other than vLLM's exact pin is present. That makes the installed tensorizer a + # file the scenario can read, so each message is asserted together with the + # state it describes. No GPU, network or real vLLM is needed. Linux only: the + # stand-ins are shell scripts and the install path is not reached on Windows. + @id:vllm-tensorizer-upgraded-when-pin-cannot-import @requires-os:linux + Scenario: vllm-tensorizer-01 - An unimportable tensorizer pin is upgraded and reported as a deliberate divergence + Given a managed vLLM runtime whose pinned tensorizer cannot be imported + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the user is told tensorizer was upgraded to 2.12.1 + And the dependency check reports the tensorizer pin as an expected divergence + And tensorizer 2.12.1 is what the runtime now holds + + @id:vllm-tensorizer-second-install-leaves-upgrade-alone @requires-os:linux + Scenario: vllm-tensorizer-02 - Installing again keeps the upgraded tensorizer and does not reinstall vLLM + Given a managed vLLM runtime whose pinned tensorizer cannot be imported + And the user has installed the vLLM engine from that runtime + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the dependency check reports the tensorizer pin as an expected divergence + And tensorizer was upgraded only once and vLLM was not reinstalled + + # An upgrade can fail to help. On a runtime this run did not install vLLM into + # that must not turn a previously successful install into a failure, but the + # warning says what will happen at serve time. + @id:vllm-tensorizer-warns-when-upgrade-does-not-help @requires-os:linux + Scenario: vllm-tensorizer-03 - An upgrade that does not help only warns on a runtime this run did not install into + Given a managed vLLM runtime whose tensorizer cannot be imported even after an upgrade + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the install warns that tensorizer still cannot be imported and shows the KeyError + + # Python 3.13+ writes a coloured traceback under FORCE_COLOR even when piped, + # with escapes around the quoted path. The probe still has to see tensorizer's + # frame, or a broken tensorizer reads as fine and `rocm serve` crashes. + @id:vllm-tensorizer-coloured-traceback-still-upgraded @requires-os:linux + Scenario: vllm-tensorizer-04 - A coloured traceback does not hide a broken tensorizer + Given a managed vLLM runtime whose Python writes its tracebacks in colour + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the user is told tensorizer was upgraded to 2.12.1 + And tensorizer 2.12.1 is what the runtime now holds + + # Offline, the upgrade cannot be fetched. On a runtime this run did not install + # vLLM into that must not turn a previously successful install into a failure, + # but the warning says what will happen at serve time and nothing was changed. + @id:vllm-tensorizer-failed-upgrade-warns-on-existing-runtime @requires-os:linux + Scenario: vllm-tensorizer-05 - A tensorizer that cannot be upgraded only warns on an existing runtime + Given a managed vLLM runtime whose pinned tensorizer cannot be imported and cannot be upgraded + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the install warns that tensorizer could not be upgraded + + # The check fails hard whenever the run installed vLLM (a first install, + # `--reinstall` or a repair), because the runtime would be left unable to serve. + # `--reinstall` on an existing runtime is the form this scenario runs. + @id:vllm-tensorizer-failed-upgrade-refuses-reinstall @requires-os:linux + Scenario: vllm-tensorizer-06 - A tensorizer that cannot be upgraded fails an install that reinstalls vLLM + Given a managed vLLM runtime whose pinned tensorizer cannot be imported and cannot be upgraded + When the user reinstalls the vLLM engine from that runtime + Then the install is refused because tensorizer could not be upgraded + + # A ROCm 10.x discover runtime launches the server with a PYTHONPATH, and + # `python -I` would drop it: the probe would then stop on a missing module + # before tensorizer and call a broken tensorizer fine. + @id:vllm-tensorizer-probe-keeps-the-server-pythonpath @requires-os:linux + Scenario: vllm-tensorizer-07 - The probe runs with the PYTHONPATH the server launches with + Given a managed vLLM runtime that only imports with the server's PYTHONPATH + When the user installs the vLLM engine from that runtime + Then the install succeeds + And the user is told tensorizer was upgraded to 2.12.1 + And tensorizer 2.12.1 is what the runtime now holds + + # An import that fails in torch, below a tensorizer frame, on a missing library + # is for the device check to report. Upgrading tensorizer would not help and + # refusing the install over it would be wrong. + @id:vllm-tensorizer-library-failure-is-left-alone @requires-os:linux + Scenario: vllm-tensorizer-08 - A missing library below tensorizer does not trigger an upgrade + Given a managed vLLM runtime whose import fails on a missing library below tensorizer + When the user installs the vLLM engine from that runtime + Then the install succeeds without changing tensorizer + + @id:vllm-tensorizer-library-error-in-tensorizer-is-left-alone @requires-os:linux + Scenario: vllm-tensorizer-09 - A missing-library error raised by tensorizer's own code does not trigger an upgrade + Given a managed vLLM runtime whose tensorizer code raises a missing-library error + When the user installs the vLLM engine from that runtime + Then the install succeeds without changing tensorizer + + @id:vllm-tensorizer-non-library-error-below-tensorizer-is-left-alone @requires-os:linux + Scenario: vllm-tensorizer-10 - A non-library exception raised below tensorizer does not trigger an upgrade + Given a managed vLLM runtime whose import raises a non-library error in torch below tensorizer + When the user installs the vLLM engine from that runtime + Then the install succeeds without changing tensorizer + + @id:vllm-tensorizer-still-broken-after-reinstall-is-refused @requires-os:linux + Scenario: vllm-tensorizer-11 - A reinstall whose tensorizer still cannot be imported says vLLM was installed + Given a managed vLLM runtime whose tensorizer cannot be imported even after an upgrade + When the user reinstalls the vLLM engine from that runtime + Then the install is refused + And the refusal says the install ran and tensorizer still cannot be imported after the upgrade + + # The remedy the CLI names beside a tensorizer divergence is a vLLM reinstall. + # A reinstall puts vLLM's own tensorizer pin back, so it only clears the + # condition if the install then upgrades tensorizer again. This starts from a + # genuine violation beside the already upgraded tensorizer and checks the final + # state: the violation gone, and tensorizer upgraded again after vLLM. + @id:vllm-tensorizer-repair-clears-violation-and-keeps-upgrade @requires-os:linux + Scenario: vllm-tensorizer-12 - Repairing a genuine violation beside the upgraded tensorizer keeps the upgrade + Given a managed vLLM runtime with a genuine violation beside the upgraded tensorizer + When the user installs the vLLM engine from that runtime + Then the violation is cleared and tensorizer is upgraded again after vLLM was reinstalled + + # The remedy the CLI names beside a tensorizer divergence is a vLLM reinstall, + # and the command run here is the one it printed. The reinstall puts vLLM's own + # tensorizer pin back, so it only clears the condition if the install then + # upgrades tensorizer again: the final state is the violation gone and + # tensorizer upgraded after vLLM was reinstalled. + @id:vllm-tensorizer-advised-reinstall-clears-violation @requires-os:linux + Scenario: vllm-tensorizer-13 - The advised reinstall clears a violation beside the upgraded tensorizer and keeps the upgrade + Given a managed vLLM runtime whose violation beside the upgraded tensorizer only a reinstall clears + When the user installs the vLLM engine from that runtime + Then the install reports the violation and advises reinstalling vLLM + When the user runs the advised command + Then the advised reinstall cleared the violation and tensorizer was upgraded after it + + # The first install: vLLM is not in the runtime yet, so this run installs it, + # and a tensorizer that cannot be upgraded fails the install, saying so. + @id:vllm-tensorizer-failed-upgrade-refuses-first-install @requires-os:linux + Scenario: vllm-tensorizer-14 - A tensorizer that cannot be upgraded fails the first install of vLLM + Given a managed runtime without vLLM whose pinned tensorizer cannot be imported and cannot be upgraded + When the user installs the vLLM engine from that runtime + Then the install is refused because tensorizer could not be upgraded diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index d63549166f..1b0cf7f332 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -16,7 +16,7 @@ //! changes engine support, so the unit tests below guard it. //! //! The AMD GPU *count* is re-implemented for the same reason (see -//! [`probe_amd_gpu_count`]): no `rocm` command reports one, so a scenario whose +//! `probe_amd_gpu_count`): no `rocm` command reports one, so a scenario whose //! premise is a multi-GPU host has nothing else to gate on. //! //! `examine`'s `default_engine` now reports the host's real engine rather than the @@ -28,10 +28,12 @@ use std::sync::OnceLock; -/// Name of the `rocm` binary under test (mirrors the test binary's -/// `rocm_binary()`): `ROCM_CLI_BINARY` or plain `rocm`. -fn rocm_binary() -> String { - std::env::var("ROCM_CLI_BINARY").unwrap_or_else(|_| "rocm".to_string()) +/// Name of the `rocm` binary under test: `ROCM_CLI_BINARY` or plain `rocm`, +/// read through `env` so a test can supply it without a process-wide variable. +/// Like the test binary's `rocm_binary()`, except that a value that is not +/// UTF-8 is used as given here rather than replaced by `rocm`. +fn rocm_binary(env: &impl Fn(&str) -> Option) -> std::ffi::OsString { + env("ROCM_CLI_BINARY").unwrap_or_else(|| "rocm".into()) } /// Whether a container runtime is usable here. @@ -132,11 +134,11 @@ pub struct HostCapability { pub gfx_target: Option, /// Whether an AMD GPU is present AND usable here, so `@requires-gpu` /// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm - /// driver path on WSL — see [`host_has_usable_gpu`]. + /// driver path on WSL — see `host_has_usable_gpu`. pub has_amd_gpu: bool, /// How many AMD GPUs are PRESENT on this host, before any visibility mask. /// `None` where the count could not be determined (non-Linux, unreadable - /// sysfs). Gates `@requires-multi-gpu`; see [`probe_amd_gpu_count`]. + /// sysfs). Gates `@requires-multi-gpu`; see `probe_amd_gpu_count`. pub amd_gpu_count: Option, /// Engine adapters the binary reports as present. Both builtins are always /// "built-in", so this is NOT the same as "can start here" — use @@ -148,22 +150,33 @@ pub struct HostCapability { /// "mock" (no AMD GPU), else the family/target (e.g. "mi300x", "strix-halo"). pub platform_slug: String, /// Whether the `rocm` binary under test carries the test-only - /// `e2e-oom-fault-injection` hook, so `@requires-oom-fault-injection` - /// scenarios can simulate a GPU-less OOM launch. It is compiled out of the - /// shipping binary, and the harness cannot probe for it (no product command + /// `e2e-oom-fault-injection` hook, so the scenarios tagged + /// `@requires-oom-fault-injection` can simulate a GPU-less launch. It is + /// compiled out of the shipping binary, and the harness cannot probe for it (no product command /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with /// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted - /// lanes' shipping release build). See [`probe_host_capability`]. + /// lanes' shipping release build). See `probe_host_capability`. pub oom_fault_injection: bool, /// Whether two file names differing only in letter case are two files under /// the temp root the scenarios' isolated roots are created in. Gates - /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`]. + /// `@requires-case-sensitive-fs`; see `probe_case_sensitive_fs`. pub case_sensitive_fs: bool, /// Whether a directory made read-only refuses new files to the user /// running the suite. Gates `@requires-dir-permissions`; see - /// [`probe_dir_permissions_bind`]. + /// `probe_dir_permissions_bind`. pub dir_permissions_bind: bool, + /// `examine`'s `driver_status:` line, empty when it printed none. Only + /// decides GPU usability on WSL (see `host_has_usable_gpu`); kept so a + /// lane whose GPU went missing can say why. + pub driver_status: String, + /// Why the probe could not read the binary under test, if it could not: + /// the first failure of `rocm examine` or `rocm engines list` (see + /// [`ProbeError`]). When set, the rest of this struct holds the same + /// no-GPU values an empty `examine` would give, so callers never panic, and + /// the harness refuses to run (see `lane_verdict::preflight_verdict`). + #[serde(skip_serializing_if = "Option::is_none")] + pub probe_error: Option, } /// Environment variable `xtask e2e` sets to `1` when it compiled the binary @@ -258,7 +271,11 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi // OS distro from `examine` — ALWAYS collected (every platform, incl. mock and // hosts without a shared runtime dir). Isolated throwaway root reads the host. if let Ok(tmp) = tempfile::TempDir::with_prefix("rocm-e2e-ver-") { - let examine = run_probe(tmp.path(), &["examine"]); + // Best-effort, unlike the capability probe: a failed `examine` here + // only leaves the report heading's OS blank. The capability probe ran + // the same command before any scenario and already refused the run if + // it failed. + let examine = run_probe(tmp.path(), &["examine"]).unwrap_or_default(); v.os = examine .lines() .find_map(|l| l.trim().strip_prefix("distro:")) @@ -486,6 +503,29 @@ fn probe_host_capability() -> HostCapability { // what the scenarios see. let examine = run_probe(root, &["examine"]); let engines = run_probe(root, &["engines", "list"]); + capability_from_probes(examine, engines, root) +} + +/// Build a [`HostCapability`] from the two probes' results; `root` is the +/// throwaway directory the filesystem probes write under. +/// +/// Split from [`probe_host_capability`] so a test can hand it a failed probe +/// without a binary to run. +fn capability_from_probes( + examine: Result, + engines: Result, + root: &std::path::Path, +) -> HostCapability { + // The first failure is the one reported; either one means the facts below + // were not read from the binary, so they fall back to the no-GPU values an + // empty dump parses to. + let probe_error = examine + .as_ref() + .err() + .or_else(|| engines.as_ref().err()) + .map(ToString::to_string); + let examine = examine.unwrap_or_default(); + let engines = engines.unwrap_or_default(); let ExamineFacts { os_family, @@ -516,6 +556,8 @@ fn probe_host_capability() -> HostCapability { oom_fault_injection, case_sensitive_fs, dir_permissions_bind, + driver_status, + probe_error, } } @@ -581,20 +623,455 @@ fn probe_case_sensitive_fs(dir: &std::path::Path) -> bool { probe().unwrap_or(false) } -/// Run `rocm ` with an isolated config/data/cache root, returning stdout -/// (empty string on any failure — the probe must never panic the suite). -fn run_probe(root: &std::path::Path, args: &[&str]) -> String { - let mut cmd = std::process::Command::new(rocm_binary()); +/// Why a probe of the binary under test produced nothing the harness can read. +/// +/// Every variant is fatal to the run unless the caller ignores it, as +/// `collect_versions` does: an unreadable probe parses as a host with no GPU, +/// so every capability-gated scenario would resolve to skip and the run would +/// pass without testing them. `command` is the binary and arguments as +/// run, so the message names exactly what failed. +#[derive(Debug)] +pub enum ProbeError { + /// The binary could not be started at all (wrong path, not executable). + Spawn { + command: String, + source: std::io::Error, + }, + /// It ran and exited non-zero: the CLI exits 0 for every finding, so this + /// means it could not examine the host at all. + Exit { + command: String, + status: std::process::ExitStatus, + stderr_tail: String, + }, + /// It exited 0 but printed nothing on stdout. + EmptyOutput { command: String }, + /// It was still running when the deadline ([`PROBE_TIMEOUT_ENV`]) passed, + /// so it was killed and reaped. `stderr_tail` is what it had written to + /// stderr by then. + Timeout { + command: String, + after: std::time::Duration, + stderr_tail: String, + }, + /// It started, but waiting for it to exit failed. + Wait { + command: String, + source: std::io::Error, + }, + /// It exited 0, but its stdout was still open `grace` later: a process it + /// started is holding the pipe, so what it printed cannot be read in full. + OutputNotClosed { + command: String, + grace: std::time::Duration, + stderr_tail: String, + }, +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Spawn { command, source } => write!( + f, + "could not start `{command}`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY" + ), + Self::Exit { + command, + status, + stderr_tail, + } => write!( + f, + "`{command}` exited with {status}\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::Timeout { + command, + after, + stderr_tail, + } => write!( + f, + "`{command}` was still running after {after:?}, so it was killed; set \ + {PROBE_TIMEOUT_ENV} to allow it longer\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::Wait { command, source } => write!( + f, + "`{command}` was started, but waiting for it to exit failed: {source}" + ), + Self::OutputNotClosed { + command, + grace, + stderr_tail, + } => write!( + f, + "`{command}` exited 0, but its stdout was still open {grace:?} later (a \ + process it started may be holding it), so what it printed cannot be read \ + in full\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::EmptyOutput { command } => { + write!(f, "`{command}` exited 0 but produced no output on stdout") + } + } + } +} + +impl std::error::Error for ProbeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Spawn { source, .. } | Self::Wait { source, .. } => Some(source), + Self::Exit { .. } + | Self::EmptyOutput { .. } + | Self::Timeout { .. } + | Self::OutputNotClosed { .. } => None, + } + } +} + +/// How many trailing stderr lines a [`ProbeError`] keeps. +const PROBE_STDERR_TAIL_LINES: usize = 20; + +/// The last [`PROBE_STDERR_TAIL_LINES`] lines of a probe's stderr. +fn stderr_tail(stderr: &[u8]) -> String { + let stderr = String::from_utf8_lossy(stderr); + let lines: Vec<&str> = stderr.lines().collect(); + lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n") +} + +/// Turn one probe's spawn result into its stdout, or the reason it has none. +/// +/// Pure, so each outcome is unit-tested without a binary to run. Stdout that is +/// only whitespace counts as empty: no `examine` field can be parsed from it. +pub fn classify_probe_output( + command: &str, + result: std::io::Result, +) -> Result { + let out = result.map_err(|source| ProbeError::Spawn { + command: command.to_owned(), + source, + })?; + if !out.status.success() { + return Err(ProbeError::Exit { + command: command.to_owned(), + status: out.status, + stderr_tail: stderr_tail(&out.stderr), + }); + } + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + if stdout.trim().is_empty() { + return Err(ProbeError::EmptyOutput { + command: command.to_owned(), + }); + } + Ok(stdout) +} + +/// How a bounded probe run ended (see `output_within`). +#[derive(Debug)] +pub enum Finished { + /// The child exited. `stdout_drained` is false when its stdout was still + /// open after the drain grace; `output` then holds what had been read by + /// then. Stderr is read the same way, but one left open does not fail the + /// probe: it carries only diagnostics. + Exited { + output: std::process::Output, + stdout_drained: bool, + }, + /// The child was still running at the deadline and has been killed and + /// reaped; `stderr` is what it had written by then. + TimedOut { stderr: Vec }, + /// Waiting for the child failed; it has been killed. + WaitFailed(std::io::Error), +} + +/// [`classify_probe_output`] for a bounded run: a timeout and an output pipe +/// left open are errors of their own, never read as "no output". +pub fn classify_finished( + command: &str, + result: std::io::Result, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> Result { + match result { + Err(source) => classify_probe_output(command, Err(source)), + Ok(Finished::TimedOut { stderr }) => Err(ProbeError::Timeout { + command: command.to_owned(), + after: timeout, + stderr_tail: stderr_tail(&stderr), + }), + Ok(Finished::WaitFailed(source)) => Err(ProbeError::Wait { + command: command.to_owned(), + source, + }), + Ok(Finished::Exited { + output, + stdout_drained: false, + }) if output.status.success() => Err(ProbeError::OutputNotClosed { + command: command.to_owned(), + grace, + stderr_tail: stderr_tail(&output.stderr), + }), + Ok(Finished::Exited { output, .. }) => classify_probe_output(command, Ok(output)), + } +} + +/// Run `rocm ` with an isolated config/data/cache root and return its +/// stdout, or why there is none. Never panics: the caller decides whether a +/// failure is fatal. +fn run_probe(root: &std::path::Path, args: &[&str]) -> Result { + run_probe_using(root, args, |name| std::env::var_os(name)) +} + +/// `run_probe` with the environment lookup passed in. +/// +/// The binary comes from `ROCM_CLI_BINARY` and the deadline from +/// [`PROBE_TIMEOUT_ENV`], both read through `env`, so a test can drive both +/// without a process-wide variable. +pub fn run_probe_using( + root: &std::path::Path, + args: &[&str], + env: impl Fn(&str) -> Option, +) -> Result { + run_probe_with( + &rocm_binary(&env), + root, + args, + probe_timeout_from_env(env(PROBE_TIMEOUT_ENV).as_deref()), + ) +} + +/// Environment variable that overrides how long one probe command may run, +/// in whole seconds. +pub const PROBE_TIMEOUT_ENV: &str = "E2E_PROBE_TIMEOUT_SECS"; + +/// How long one probe command may run when [`PROBE_TIMEOUT_ENV`] is unset. +/// +/// `rocm examine` normally answers in seconds; this is generous so a loaded +/// runner does not fail a good probe, while a wedged one still fails the lane +/// with a named reason long before the job's own timeout. +pub const DEFAULT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2); + +/// The probe deadline from [`PROBE_TIMEOUT_ENV`]'s value. +/// +/// Accepts any whole number of seconds of at least 1, with surrounding +/// whitespace ignored; a value too large to add to the current time is waited +/// on without a deadline (see `output_within`). Anything else (unset, empty, +/// zero, negative, a fraction, a non-number, a number past `u64`, or a value +/// that is not UTF-8) gives [`DEFAULT_PROBE_TIMEOUT`] rather than an error. +#[must_use] +pub fn probe_timeout_from_env(value: Option<&std::ffi::OsStr>) -> std::time::Duration { + value + .and_then(std::ffi::OsStr::to_str) + .and_then(|v| v.trim().parse::().ok()) + .filter(|secs| *secs >= 1) + .map_or(DEFAULT_PROBE_TIMEOUT, std::time::Duration::from_secs) +} + +/// `run_probe` with the binary and deadline passed in, so a test can name +/// them without setting process-wide environment variables. +pub fn run_probe_with( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], + timeout: std::time::Duration, +) -> Result { + run_probe_bounded(binary, root, args, timeout, PROBE_DRAIN_GRACE) +} + +/// [`run_probe_with`] with the drain grace passed in too, for tests. +fn run_probe_bounded( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], + timeout: std::time::Duration, + grace: std::time::Duration, +) -> Result { + let mut cmd = std::process::Command::new(binary); cmd.args(args); cmd.env("ROCM_CLI_CONFIG_DIR", root.join("config")); cmd.env("ROCM_CLI_DATA_DIR", root.join("data")); cmd.env("ROCM_CLI_CACHE_DIR", root.join("cache")); - match cmd.output() { - Ok(out) => String::from_utf8_lossy(&out.stdout).into_owned(), - Err(_) => String::new(), + let command = std::iter::once(binary.to_string_lossy().into_owned()) + .chain(args.iter().map(|a| (*a).to_owned())) + .collect::>() + .join(" "); + classify_finished(&command, output_within(cmd, timeout, grace), timeout, grace) +} + +/// How often [`output_within`] checks whether the child has exited. +const PROBE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); + +/// How long [`output_within`] waits, after the child exited or was killed, for +/// its output pipes to close. Only a process it started that inherited a pipe +/// and outlives it can hold one open past this. +const PROBE_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + +/// A pipe being read on its own thread into a buffer the caller can read at any +/// time, so bytes read before a timeout or a stuck drain are not lost. +struct PipeReader { + bytes: std::sync::Arc>>, + done: std::sync::mpsc::Receiver<()>, +} + +impl PipeReader { + fn spawn(pipe: Option) -> Self { + let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let (sender, done) = std::sync::mpsc::channel(); + let sink = std::sync::Arc::clone(&bytes); + std::thread::spawn(move || { + if let Some(mut pipe) = pipe { + let mut chunk = [0u8; 8192]; + loop { + match pipe.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(n) => sink + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(&chunk[..n]), + } + } + } + let _ = sender.send(()); + }); + Self { bytes, done } + } + + /// Wait until the pipe closes or `deadline` passes; whether it closed. + fn wait_closed(&self, deadline: std::time::Instant) -> bool { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + self.done.recv_timeout(left).is_ok() + } + + /// What has been read so far. + fn snapshot(&self) -> Vec { + self.bytes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() } } +/// `command.output()`, bounded by `timeout`: a child still running then is +/// killed and reaped. Either way the output pipes get `grace` to close, and +/// whatever was read by then is kept. +/// +/// std-only, polling `try_wait`, so it behaves the same on Linux and Windows, +/// with each output pipe read on its own thread so a chatty child cannot block +/// on a full pipe. It repeats the spawn, poll, kill and reap steps of +/// `device_memory::bounded_stdout`; a change to the kill or drain behaviour of +/// one should look at the other. It differs in three ways: +/// +/// - it keeps stderr and the exit status, which the probe reports; +/// - it drains after a kill too, where `bounded_stdout` gives up at once, +/// because the stderr written before the deadline is the only evidence of +/// why `rocm examine` hung; +/// - its grace is 5 s (`PROBE_DRAIN_GRACE`) rather than 2 s, because it runs +/// a handful of times per suite: each probe runs once (the capability +/// probes in `probe_host_capability`, cached by `host_capability`, and the +/// version probe in `collect_versions`, whose `rocm examine` repeats the +/// capability probe's), not once per sample, so a generous wait for a +/// loaded runner to hand over the bytes costs little. +/// +/// Only the direct child is killed: a process it started that inherited a pipe +/// can keep that pipe open past the grace. The reader thread for that pipe is +/// then left detached until the pipe closes, and the bytes it had read are +/// used as they are. A leaked thread per stuck probe is accepted in a test +/// harness that runs a handful of probes, each once, and exits. +fn output_within( + mut command: std::process::Command, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> std::io::Result { + use std::process::Stdio; + + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout = PipeReader::spawn(child.stdout.take()); + let stderr = PipeReader::spawn(child.stderr.take()); + + // `None` only for a deadline too far off to represent: wait without one. + let deadline = std::time::Instant::now().checked_add(timeout); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if deadline.is_none_or(|deadline| std::time::Instant::now() < deadline) => { + std::thread::sleep(PROBE_POLL_INTERVAL); + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Ok(Finished::WaitFailed(error)); + } + } + }; + let drained_by = std::time::Instant::now() + grace; + let stdout_closed = stdout.wait_closed(drained_by); + // Waited for so the tail is complete when it can be, but not required. + let _ = stderr.wait_closed(drained_by); + Ok(match status { + None => Finished::TimedOut { + stderr: stderr.snapshot(), + }, + Some(status) => Finished::Exited { + output: std::process::Output { + status, + stdout: stdout.snapshot(), + stderr: stderr.snapshot(), + }, + stdout_drained: stdout_closed, + }, + }) +} + +/// One line naming the probe facts that decide GPU usability, for a lane whose +/// declared GPU the probe did not find. +/// +/// Reads the visibility masks from the environment the way +/// `host_has_usable_gpu` does; [`probe_facts_summary_with_mask`] is the pure +/// form. +#[must_use] +pub fn probe_facts_summary(cap: &HostCapability) -> String { + let hip = std::env::var_os("HIP_VISIBLE_DEVICES"); + let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES"); + probe_facts_summary_with_mask(cap, hip.as_deref(), rocr.as_deref()) +} + +/// Pure form of [`probe_facts_summary`]. Names the mask that is in force by the +/// same precedence as `selected_visibility_mask`: HIP wins when set, even +/// empty. +#[must_use] +pub fn probe_facts_summary_with_mask( + cap: &HostCapability, + hip: Option<&std::ffi::OsStr>, + rocr: Option<&std::ffi::OsStr>, +) -> String { + let mask = match (hip, rocr) { + (Some(v), _) => format!("HIP_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, Some(v)) => format!("ROCR_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, None) => "no visibility mask".to_owned(), + }; + let driver_status = if cap.driver_status.is_empty() { + "" + } else { + cap.driver_status.as_str() + }; + format!( + "detected_gfx_target={}, wsl={}, driver_status={driver_status}, {mask}", + cap.gfx_target.as_deref().unwrap_or(""), + cap.is_wsl, + ) +} + /// The `examine` fields the capability probe reads. struct ExamineFacts { os_family: String, @@ -992,6 +1469,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(strix.engine_available("lemonade")); // vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu. @@ -1009,6 +1488,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(mi300x.engine_available("vllm")); assert!(mi300x.engine_available("lemonade")); @@ -1442,9 +1923,9 @@ Local model engines /// `expectation.rs`'s `requires_oom_fault_injection_runs_only_when_the_hook_is_built_in` /// builds `HostCapability { oom_fault_injection: .. }` by hand and so never /// reaches this comparison; without this test, editing the literal on one - /// side of the producer/consumer pair would make serve-41 a permanent skip - /// with nothing red. The `"1"` asserted here is the same literal - /// `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned + /// side of the producer/consumer pair would make the scenarios tagged + /// `@requires-oom-fault-injection` a permanent skip with nothing red. The + /// `"1"` asserted here is the same literal `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned /// by `release_build_resolves_and_exports_both_cli_binaries` on that side. #[test] fn only_the_exact_value_the_producer_writes_arms_the_hook() { @@ -1503,4 +1984,667 @@ Local model engines std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore"); assert_eq!(probe_dir_permissions_bind(tmp.path()), refused); } + + /// An exit status that reports failure with code 1, built the way each + /// platform encodes it, so the non-zero-exit row runs on both CI gates. + fn failed_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + // A wait status: the exit code sits in the second byte. + std::process::ExitStatus::from_raw(1 << 8) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(1) + } + } + + fn succeeded_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + } + + fn output( + status: std::process::ExitStatus, + stdout: &str, + stderr: &str, + ) -> std::process::Output { + std::process::Output { + status, + stdout: stdout.as_bytes().to_vec(), + stderr: stderr.as_bytes().to_vec(), + } + } + + /// The regression: a probe that exits non-zero is an error even when it + /// printed something, rather than stdout parsed as a GPU-less host. + #[test] + fn a_probe_that_exits_non_zero_is_an_error_with_its_stderr_tail() { + let result = classify_probe_output( + "rocm examine", + Ok(output( + failed_status(), + "rocm examine\n os: linux\n", + "warning: early\nError: could not read the host\n", + )), + ); + let Err(error @ ProbeError::Exit { .. }) = result else { + panic!("a non-zero exit must be ProbeError::Exit, got {result:?}"); + }; + assert_eq!( + error.to_string(), + format!( + "`rocm examine` exited with {}\n--- stderr (last lines) ---\n\ + warning: early\nError: could not read the host", + failed_status() + ) + ); + } + + #[test] + fn the_stderr_tail_keeps_only_the_last_lines() { + let stderr = (1..=30) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let Err(ProbeError::Exit { stderr_tail, .. }) = + classify_probe_output("rocm examine", Ok(output(failed_status(), "", &stderr))) + else { + panic!("a non-zero exit must be ProbeError::Exit"); + }; + let expected: Vec = (11..=30).map(|i| format!("line {i}")).collect(); + assert_eq!(stderr_tail, expected.join("\n")); + } + + #[test] + fn a_probe_that_exits_zero_with_output_returns_its_stdout() { + let stdout = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n"; + assert_eq!( + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))) + .expect("exit 0 with output is a reading"), + stdout + ); + } + + #[test] + fn a_probe_that_exits_zero_with_no_output_is_an_error() { + for stdout in ["", " \n\n"] { + let result = + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))); + let Err(error @ ProbeError::EmptyOutput { .. }) = result else { + panic!("empty stdout {stdout:?} must be ProbeError::EmptyOutput, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` exited 0 but produced no output on stdout" + ); + } + } + + #[test] + fn a_probe_whose_binary_does_not_exist_is_a_spawn_error_naming_it() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let missing = tmp.path().join("no-such-rocm"); + let result = run_probe_with( + missing.as_os_str(), + tmp.path(), + &["examine"], + DEFAULT_PROBE_TIMEOUT, + ); + let Err(error @ ProbeError::Spawn { .. }) = result else { + panic!("a missing binary must be ProbeError::Spawn, got {result:?}"); + }; + let ProbeError::Spawn { command, source } = &error else { + unreachable!() + }; + assert_eq!(command, &format!("{} examine", missing.display())); + assert_eq!(source.kind(), std::io::ErrorKind::NotFound); + assert_eq!( + error.to_string(), + format!( + "could not start `{} examine`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY", + missing.display() + ) + ); + } + + const GPU_EXAMINE: &str = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n wsl: false\n driver_status: ok\n"; + + /// A failed probe records why, and leaves the struct at the no-GPU values + /// so no caller panics. + #[test] + fn a_failed_examine_records_the_error_and_falls_back_to_no_gpu() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Ok("lemonade\n".to_owned()), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + assert!(!cap.has_amd_gpu); + assert_eq!(cap.gfx_target, None); + assert_eq!(cap.platform_slug, "mock"); + } + + /// The `engines list fails, examine OK` row: the GPU facts were read, but + /// the probe still failed, so the run must not proceed on them. + #[test] + fn a_failed_engines_list_is_recorded_even_when_examine_succeeded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Err(ProbeError::Exit { + command: "rocm engines list".to_owned(), + status: failed_status(), + stderr_tail: "boom".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error, + Some(format!( + "`rocm engines list` exited with {}\n--- stderr (last lines) ---\nboom", + failed_status() + )) + ); + assert!(cap.available_engines.is_empty()); + } + + #[test] + fn the_first_failure_is_the_one_recorded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Err(ProbeError::EmptyOutput { + command: "rocm engines list".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + } + + #[test] + fn successful_probes_record_no_error_and_keep_the_driver_status() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Ok("lemonade\nvllm\n".to_owned()), + tmp.path(), + ); + assert_eq!(cap.probe_error, None); + assert_eq!(cap.gfx_target.as_deref(), Some("gfx1151")); + assert_eq!(cap.driver_status, "ok"); + // `probe_error` is left out of platform.json when there is none. + let json = serde_json::to_value(&cap).expect("serializes"); + assert!(json.get("probe_error").is_none(), "{json}"); + } + + #[test] + fn the_probe_facts_summary_names_the_mask_in_force() { + let cap = HostCapability { + os_family: "linux".to_owned(), + is_wsl: true, + gfx_target: Some("gfx1151".to_owned()), + has_amd_gpu: false, + amd_gpu_count: None, + available_engines: vec![], + effective_serve_engine: "lemonade".to_owned(), + platform_slug: "mock".to_owned(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: "wsl_rocdxg_missing".to_owned(), + probe_error: None, + }; + let base = "detected_gfx_target=gfx1151, wsl=true, driver_status=wsl_rocdxg_missing"; + assert_eq!( + probe_facts_summary_with_mask(&cap, None, None), + format!("{base}, no visibility mask") + ); + assert_eq!( + probe_facts_summary_with_mask( + &cap, + Some(std::ffi::OsStr::new("")), + Some(std::ffi::OsStr::new("0")) + ), + format!("{base}, HIP_VISIBLE_DEVICES=\"\"") + ); + assert_eq!( + probe_facts_summary_with_mask(&cap, None, Some(std::ffi::OsStr::new("1"))), + format!("{base}, ROCR_VISIBLE_DEVICES=\"1\"") + ); + let none = HostCapability { + gfx_target: None, + is_wsl: false, + driver_status: String::new(), + ..cap + }; + assert_eq!( + probe_facts_summary_with_mask(&none, None, None), + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask" + ); + } + + /// A shell command line, run as `(binary, args)` on each CI gate. + fn shell_line(line: &str) -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + ("sh".into(), vec!["-c".to_owned(), line.to_owned()]) + } + #[cfg(windows)] + { + ("cmd".into(), vec!["/C".to_owned(), line.to_owned()]) + } + } + + fn arg_refs(args: &[String]) -> Vec<&str> { + args.iter().map(String::as_str).collect() + } + + /// Prints `done` after about two seconds. + fn two_second_command() -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + shell_line("sleep 2; echo done") + } + #[cfg(windows)] + { + shell_line("ping -n 3 127.0.0.1 >nul & echo done") + } + } + + /// The env lookup `run_probe` would get, with only the given variables set. + fn env_of( + vars: Vec<(&'static str, std::ffi::OsString)>, + ) -> impl Fn(&str) -> Option { + move |name| { + vars.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + } + } + + /// `E2E_PROBE_TIMEOUT_SECS` reaches the probe through the same lookup + /// `run_probe` uses (no process-wide variable), and the Timeout message's + /// advice holds: the same command that timed out at 1 second completes when + /// the variable allows it longer. + #[test] + fn the_probe_timeout_variable_sets_the_deadline_and_raising_it_clears_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = two_second_command(); + let command = format!("{} {}", binary.to_string_lossy(), args.join(" ")); + + let short = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary.clone()), + (PROBE_TIMEOUT_ENV, "1".into()), + ]), + ); + let Err(error @ ProbeError::Timeout { .. }) = short else { + panic!("a 1 s deadline on a 2 s command must time out, got {short:?}"); + }; + let ProbeError::Timeout { after, .. } = &error else { + unreachable!() + }; + assert_eq!(*after, std::time::Duration::from_secs(1)); + assert_eq!( + error.to_string(), + format!( + "`{command}` was still running after 1s, so it was killed; set \ + E2E_PROBE_TIMEOUT_SECS to allow it longer\n--- stderr (last lines): (empty) ---" + ) + ); + + // Following that advice clears the condition. + let longer = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary), + (PROBE_TIMEOUT_ENV, "5".into()), + ]), + ) + .expect("with 5 s allowed, the 2 s command completes"); + assert_eq!(longer.trim(), "done"); + } + + /// The regression: a probe that never finishes must become an error at its + /// deadline, carrying what it had written to stderr, and be killed AND + /// reaped. A zombie still answers `kill -0`, so a failing `kill -0` shows + /// the child was reaped, not merely signalled. + #[cfg(unix)] + #[test] + fn a_probe_still_running_at_its_deadline_is_killed_reaped_and_keeps_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let pid_file = tmp.path().join("pid"); + let script = format!( + "echo $$ > '{}'; echo early-err >&2; exec sleep 30", + pid_file.display() + ); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", &script], + std::time::Duration::from_secs(3), + std::time::Duration::from_secs(2), + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(30), + "returned after {:?}: it waited for the child instead of killing it", + started.elapsed() + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "early-err"); + // The shell creates the file and then writes the pid; give a loaded + // runner a moment to finish the write rather than read it half-done. + let mut pid = String::new(); + for _ in 0..100 { + pid = std::fs::read_to_string(&pid_file).unwrap_or_default(); + if !pid.trim().is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!( + !pid.trim().is_empty(), + "the child wrote its pid before the deadline" + ); + let still_there = std::process::Command::new("kill") + .args(["-0", pid.trim()]) + .stderr(std::process::Stdio::null()) + .status() + .expect("kill runs") + .success(); + assert!( + !still_there, + "child {} was left running or unreaped", + pid.trim() + ); + } + + /// Windows half of the test above: the deadline and the stderr kept from + /// before it. `cmd` is killed but the `ping` it started holds the pipe, so + /// the bytes reach the caller only through the shared buffer. The reaping + /// check needs `kill -0` and runs on Unix only. + #[cfg(windows)] + #[test] + fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul"); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + ); + assert!(started.elapsed() < std::time::Duration::from_secs(10)); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail.trim(), "early-err"); + } + + /// Stderr written after the kill still reaches the caller: the drain runs + /// after a timeout too. A grandchild holding only stderr (its stdout goes + /// to `/dev/null`) writes `late` half a second after the 1 s deadline + /// killed its parent. Stdout is closed by then, so only the wait on stderr + /// can collect it; the 10 s grace leaves time for that. + #[cfg(unix)] + #[test] + fn stderr_written_after_the_kill_is_drained_into_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &[ + "-c", + "(sleep 1.5; echo late >&2) >/dev/null & exec sleep 30", + ], + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(10), + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "late"); + } + + /// A child that reads stdin gets EOF at once rather than blocking until the + /// deadline: `output_within` replaces whatever stdin the command carried + /// with an empty one. The command is handed a pipe whose write end this + /// test holds open, so only that replacement lets the read finish (the + /// same arrangement as `a_child_that_reads_stdin_is_given_an_empty_one` in + /// rocm-core's `process/bounded.rs`). + /// + /// Like that sibling, it catches a missing stdin setting but does not pin + /// `null` against `inherit`: an `inherit` mutant passes whenever the test + /// process's own stdin is already at EOF. + #[test] + fn a_probe_that_reads_stdin_is_given_an_empty_one() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", "read line || echo stdin-eof"]); + command + }; + #[cfg(windows)] + let mut command = { + // `set /p` leaves the variable as it was when its read hits EOF. + let script = tmp.path().join("stdin.bat"); + std::fs::write( + &script, + "@echo off\r\nset \"LINE=stdin-eof\"\r\nset /p LINE=\r\necho %LINE%\r\n", + ) + .expect("write script"); + let mut command = std::process::Command::new("cmd"); + command.arg("/C").arg(&script); + command + }; + let (stdin_reader, stdin_writer) = std::io::pipe().expect("stdin pipe"); + command.stdin(stdin_reader); + + let started = std::time::Instant::now(); + let finished = output_within( + command, + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(2), + ) + .expect("spawns"); + let elapsed = started.elapsed(); + // Dropped only now: closing the write end earlier would give the child + // EOF whatever stdin `output_within` set. + drop(stdin_writer); + drop(tmp); + + let Finished::Exited { output, .. } = finished else { + panic!("a child given an empty stdin must exit, not time out: {finished:?}"); + }; + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "stdin-eof"); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the read must hit EOF at once, but took {elapsed:?}" + ); + } + + /// A probe that exited 0 while a process it started still holds its + /// stdout is reported as such, not as "produced no output". Unix only: it + /// needs a background process that inherits the pipe, which `sh` gives + /// reliably; the classification itself is tested on both gates below. + #[cfg(unix)] + #[test] + fn a_probe_whose_output_stays_open_is_reported_as_that_not_as_no_output() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; echo warn >&2; sleep 3 &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + let Err(error @ ProbeError::OutputNotClosed { .. }) = &result else { + panic!("an output left open must be ProbeError::OutputNotClosed, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`sh -c echo real-output; echo warn >&2; sleep 3 &` exited 0, but its stdout was \ + still open 300ms later (a process it started may be holding it), so what it \ + printed cannot be read in full\n--- stderr (last lines) ---\nwarn" + ); + } + + /// Only stdout carries the reading: a process that keeps just stderr open + /// does not fail the probe. Unix only, like the test above. + #[cfg(unix)] + #[test] + fn a_probe_whose_stderr_alone_stays_open_still_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; sleep 3 >/dev/null &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + assert_eq!( + result.expect("stdout closed, so a reading").trim(), + "real-output" + ); + } + + #[test] + fn an_undrained_exit_is_classified_by_its_status() { + let grace = std::time::Duration::from_secs(5); + let undrained = |status| { + Ok(Finished::Exited { + output: output(status, "partial", "err"), + stdout_drained: false, + }) + }; + let ok = classify_finished("rocm examine", undrained(succeeded_status()), grace, grace); + assert!( + matches!(&ok, Err(ProbeError::OutputNotClosed { stderr_tail, .. }) if stderr_tail == "err"), + "{ok:?}" + ); + // A non-zero exit is reported as one, with what stderr had by then. + let failed = classify_finished("rocm examine", undrained(failed_status()), grace, grace); + assert!( + matches!(&failed, Err(ProbeError::Exit { stderr_tail, .. }) if stderr_tail == "err"), + "{failed:?}" + ); + // A drained exit is classified as before. + let drained = classify_finished( + "rocm examine", + Ok(Finished::Exited { + output: output(succeeded_status(), "rocm examine\n", "partial err"), + stdout_drained: true, + }), + grace, + grace, + ); + // Stdout closed but stderr may not have: the reading is used. + assert_eq!(drained.expect("a reading"), "rocm examine\n"); + let wait_failed = classify_finished( + "rocm examine", + Ok(Finished::WaitFailed(std::io::Error::other("no child"))), + grace, + grace, + ); + let Err(error @ ProbeError::Wait { .. }) = &wait_failed else { + panic!("a failed wait must be ProbeError::Wait, got {wait_failed:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` was started, but waiting for it to exit failed: no child" + ); + let timed_out = classify_finished( + "rocm examine", + Ok(Finished::TimedOut { + stderr: b"a\nb\n".to_vec(), + }), + grace, + grace, + ); + assert!( + matches!(&timed_out, Err(ProbeError::Timeout { stderr_tail, after, .. }) + if stderr_tail == "a\nb" && *after == grace), + "{timed_out:?}" + ); + } + + #[test] + fn a_probe_that_finishes_in_time_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo probe-ok"); + let stdout = run_probe_with(&binary, tmp.path(), &arg_refs(&args), DEFAULT_PROBE_TIMEOUT) + .expect("a quick command is a reading"); + assert_eq!(stdout.trim(), "probe-ok"); + // A deadline too far off to add to the current time must not panic. + let stdout = run_probe_with( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::MAX, + ) + .expect("an unbounded deadline still reads"); + assert_eq!(stdout.trim(), "probe-ok"); + } + + #[test] + fn the_probe_deadline_reads_whole_positive_seconds_else_the_default() { + assert_eq!(probe_timeout_from_env(None), DEFAULT_PROBE_TIMEOUT); + assert_eq!(DEFAULT_PROBE_TIMEOUT.as_secs(), 120); + for value in ["5", " 5 ", "5\n"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + std::time::Duration::from_secs(5), + "{value:?}" + ); + } + // No upper bound: a huge value is honoured, and `output_within`'s + // `checked_add` keeps it from overflowing (see the `Duration::MAX` run + // in `a_probe_that_finishes_in_time_returns_its_stdout`). + for secs in [86_401, u64::MAX] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(&secs.to_string()))), + std::time::Duration::from_secs(secs) + ); + } + for value in ["", "0", "-3", "1.5", "soon", "18446744073709551616"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + DEFAULT_PROBE_TIMEOUT, + "{value:?}" + ); + } + } } diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index fb3a513e6f..0ff33664a6 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -585,7 +585,8 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); /// module is library code, and a normal dependency would be a new first-party /// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two /// can drift, so a change to either one's kill or drain behaviour should look at -/// the other. +/// the other. The same applies to `capability::output_within`, the capability +/// probe's copy of these steps, which also keeps stderr and the exit status. pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { let mut child = match command .stdin(Stdio::null()) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6b..aa8408d0da 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -116,7 +116,8 @@ pub struct ScenarioDecl { pub requires_wsl: bool, /// `@requires-oom-fault-injection`: the scenario drives the test-only /// `e2e-oom-fault-injection` hook (armed by `ROCM_E2E_SIMULATE_OOM_LAUNCH`) - /// to fabricate a GPU-less OOM launch, so it can only run against a binary + /// to fabricate a GPU-less launch (one that OOMs, or one that becomes ready + /// and then fails its smoke test), so it can only run against a binary /// compiled with that feature. `xtask e2e` builds it in when it builds the /// binary itself (the GitHub-hosted mock lane), but the self-hosted lanes /// test a prebuilt shipping release binary that has the hook compiled out — @@ -481,6 +482,12 @@ pub struct PlatformManifest<'a> { pub capability: &'a HostCapability, /// Component versions (OS/ROCm/vLLM/lemonade) for the report heading. pub versions: crate::capability::PlatformVersions, + /// Why the post-run lane check failed the run + /// (`lane_verdict::post_run_verdict`), so the consolidated report shows the + /// lane as failed even when every scenario it resolved was a skip. Left out + /// when the check passed. + #[serde(skip_serializing_if = "Option::is_none")] + pub lane_failure: Option, pub expectations: Vec, } @@ -744,6 +751,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-ubuntu" => HostCapability { os_family: "linux".into(), @@ -757,6 +766,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-windows" => HostCapability { os_family: "windows".into(), @@ -770,6 +781,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // A WSL2 dev box with the ROCm passthrough in place: `os_family` is // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes @@ -792,6 +805,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: "wsl_rocdxg_ready".into(), + probe_error: None, }, // The same box without the passthrough: the gfx target is reported // by the Windows-side driver but ROCm cannot reach it, so the probe @@ -808,6 +823,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // The hosted WSL runner before the ROCm passthrough is complete: // Windows still reports the gfx target, but the CLI cannot use it. @@ -823,6 +840,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, _ => HostCapability { os_family: "other".into(), @@ -836,6 +855,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, } } @@ -2119,6 +2140,36 @@ reason = "unrelated open bug" )); } + /// What the lane-verdict notes say about `@requires-gpu` scenarios (see + /// `lane_verdict::preflight_verdict`): `has_amd_gpu` alone decides their + /// skip, other gates aside. `resolve` takes no lane declaration, so + /// `E2E_EXPECT_GPU` does not enter into it. + #[test] + fn requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not() { + let m = Expectations::default(); + let d = decl(&["id:gpu-premise", "requires-gpu"]); + let included = Included { + nightly: false, + lifecycle: false, + docker: false, + merge_queue: false, + }; + let with_gpu = cap("strix-ubuntu"); + assert!(with_gpu.has_amd_gpu); + assert_eq!( + resolve(&d, &with_gpu, &m, included), + Expectation::ExpectPass + ); + let without_gpu = HostCapability { + has_amd_gpu: false, + ..with_gpu + }; + assert!(matches!( + resolve(&d, &without_gpu, &m, included), + Expectation::Skip { .. } + )); + } + #[test] fn non_gpu_scenario_always_runs() { let m = Expectations::default(); @@ -2356,7 +2407,8 @@ flaky = true )); } - /// The two rows for Lemonade 11.5.1's failed backend alignment apply to native + /// The two rows for the Windows Lemonade backend problems (written while 11.5.1 + /// was pinned, kept until a nightly shows them stale) apply to native /// Windows with lemonade and to nothing else: not to Linux hosts (including one whose /// default engine is vLLM, where the engine named is what decides), and they are not /// `flaky` (an XPASS means the cause is gone and the row is stale). Neither sets diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs new file mode 100644 index 0000000000..9d58ef5378 --- /dev/null +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -0,0 +1,961 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether a run's outcome says anything about the lane it ran on. +//! +//! Scenario results alone cannot say that. Scenarios the capability probe +//! resolves to skip never reach `report.json`, so a run in which every scenario +//! was skipped, or none was selected, has no failure to report and passes. +//! That is what a GPU lane does when its probe cannot read the binary under +//! test, or when the GPU is not visible: every GPU scenario (`@requires-gpu` or +//! `@requires-multi-gpu`) resolves to skip, and the lane is green without +//! having tested a GPU. +//! +//! Two checks close that gap, both pure so every row of their state matrix is a +//! unit test: +//! +//! - [`preflight_verdict`], before any scenario runs: a probe that failed is +//! fatal on every lane, and so is a lane that declares a GPU (`1` in +//! [`EXPECT_GPU_ENV`]) where the probe found none. +//! - [`post_run_verdict`], after the run: a run that executed no scenario is +//! fatal on every lane, and so is a GPU-declaring lane on which no GPU +//! scenario (`@requires-gpu` or `@requires-multi-gpu`) executed. +//! +//! The module also holds the decisions `tests/e2e.rs` takes around those two +//! checks, so they are unit-tested too: [`preflight`] (the declaration plus +//! the pre-run verdict), [`needs_gpu`] and [`gpu_scenario_counts`] (the input to +//! the post-run check), [`run_failed`] (the final exit, combining the +//! harness's own XPASS and failure classification with the post-run verdict), +//! [`finish_run`] (the post-run verdict and the `platform.json` manifest that +//! records it), and [`failure_lines`] and [`note_line`] (the exact lines +//! printed, with the annotation escaped by [`workflow_command`]). The harness +//! itself still sets up the run, probes the host, reads its other environment +//! variables, classifies the scenario results, writes the files, and does the +//! printing and exiting; that glue has no test of its own. + +use std::ffi::OsStr; + +pub use e2e_report::EXPECT_GPU_ENV; + +/// What a lane says about its GPU in [`EXPECT_GPU_ENV`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuDeclaration { + /// `1`: the lane runs on GPU hardware, so a run that found or used none is + /// broken. + Expected, + /// `0`: the lane deliberately has no usable GPU, so `@requires-gpu` + /// scenarios are not applicable on it. + NotExpected, + /// Unset or empty: no claim either way (GitHub-hosted lanes, local runs). + Undeclared, +} + +/// Read [`EXPECT_GPU_ENV`]'s value. +/// +/// Anything other than `1`, `0`, empty or unset is an error rather than a +/// guess, since a misspelt declaration that read as "undeclared" would switch +/// the GPU check off without anyone noticing. +pub fn parse_gpu_declaration(value: Option<&OsStr>) -> Result { + match value.map(OsStr::to_str) { + None | Some(Some("")) => Ok(GpuDeclaration::Undeclared), + Some(Some("1")) => Ok(GpuDeclaration::Expected), + Some(Some("0")) => Ok(GpuDeclaration::NotExpected), + Some(_) => Err(format!( + "invalid {EXPECT_GPU_ENV} value {:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset", + value.unwrap_or_default().to_string_lossy() + )), + } +} + +/// What a check concluded: a reason to fail the run, and lines to print either +/// way. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Verdict { + /// Why the run must fail; `None` when it may proceed or pass. + pub fatal: Option, + /// Informational lines that do not change the outcome. + pub notes: Vec, +} + +/// The verdict before any scenario runs, from the capability probe and the +/// lane's declaration. +/// +/// `probe_error` is [`crate::capability::HostCapability::probe_error`]; +/// `probe_facts` is the one-line summary of the facts the GPU decision was made +/// from ([`crate::capability::probe_facts_summary`]), quoted when a declared GPU +/// is missing. +#[must_use] +pub fn preflight_verdict( + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if let Some(error) = probe_error { + verdict.fatal = Some(format!( + "the host capability probe could not read the rocm binary under test, so every \ + capability-gated scenario would resolve to skip and the run would pass without \ + testing them: {error}" + )); + return verdict; + } + match (declaration, has_amd_gpu) { + (GpuDeclaration::Expected, false) => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but the capability probe found \ + no usable AMD GPU ({probe_facts}), so every @requires-gpu scenario would \ + resolve to skip" + )); + } + (GpuDeclaration::NotExpected, true) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0) but the capability probe \ + found one ({probe_facts}); the declaration is stale, and @requires-gpu scenarios \ + will run" + )), + (GpuDeclaration::NotExpected, false) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0); @requires-gpu scenarios \ + are not applicable here" + )), + (GpuDeclaration::Expected, true) | (GpuDeclaration::Undeclared, _) => {} + } + verdict +} + +/// How many GPU scenarios (`@requires-gpu` or `@requires-multi-gpu`) the run +/// resolved and how many of those executed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GpuScenarioCounts { + /// Scenarios tagged `@requires-gpu` or `@requires-multi-gpu` that the + /// harness resolved, whether or not they ran. + pub resolved: usize, + /// Of those, how many reached `report.json`. + pub executed: usize, +} + +/// The verdict after the run. +/// +/// `executed` counts the scenarios that reached `report.json`. `gpu` is `None` +/// when the harness's scenario filter never ran — a cucumber `--name`/`--tags` +/// filter on the command line replaces it — so nothing is known about which +/// scenarios needed a GPU and only the zero-executed check applies. +#[must_use] +pub fn post_run_verdict( + executed: usize, + gpu: Option, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if executed == 0 { + verdict.fatal = Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check the \ + cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS" + .to_owned(), + ); + return verdict; + } + let Some(gpu) = gpu else { + return verdict; + }; + match declaration { + GpuDeclaration::Expected if gpu.executed == 0 => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed ({} resolved, none ran)", + gpu.resolved + )); + } + GpuDeclaration::NotExpected => verdict.notes.push(format!( + "{} of {} GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on this \ + lane ({EXPECT_GPU_ENV}=0)", + gpu.resolved.saturating_sub(gpu.executed), + gpu.resolved + )), + GpuDeclaration::Expected | GpuDeclaration::Undeclared => {} + } + verdict +} + +/// What the harness carries forward from the pre-run check when it passes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Preflight { + /// The lane's parsed [`EXPECT_GPU_ENV`] declaration, for the post-run check. + pub declaration: GpuDeclaration, + /// Lines to print before the run. + pub notes: Vec, +} + +/// The pre-run lane check, from the raw [`EXPECT_GPU_ENV`] value and the +/// probe's results: `Err` with the reason when the run must not start. +/// +/// An invalid declaration is reported before anything the probe found, since +/// the verdict depends on the declaration. The harness only prints the notes +/// or the refusal and exits. +pub fn preflight( + declaration_value: Option<&OsStr>, + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, +) -> Result { + let declaration = parse_gpu_declaration(declaration_value)?; + let verdict = preflight_verdict(probe_error, has_amd_gpu, probe_facts, declaration); + match verdict.fatal { + Some(fatal) => Err(fatal), + None => Ok(Preflight { + declaration, + notes: verdict.notes, + }), + } +} + +/// Whether a scenario counts toward the post-run GPU check. +/// +/// It does when tagged `@requires-gpu` or `@requires-multi-gpu`. `@requires-gfx-target` does not +/// count, since such a scenario reads the chip name without using the GPU. +#[must_use] +pub const fn needs_gpu(decl: &crate::expectation::ScenarioDecl) -> bool { + decl.requires_gpu || decl.requires_multi_gpu +} + +/// Count the GPU scenarios the harness resolved and how many of them executed. +/// +/// `resolved` is each scenario the harness's filter saw, as `(id, needs_gpu)`; +/// `executed` is the id of each scenario in `report.json`. `None` when nothing +/// was resolved: a cucumber `--name`/`--tags` filter replaced the harness's +/// filter, so which scenarios need a GPU is unknown (see [`post_run_verdict`]). +/// An executed id the harness did not resolve as a GPU scenario is not counted. +pub fn gpu_scenario_counts<'a>( + resolved: impl IntoIterator, + executed: impl IntoIterator, +) -> Option { + let mut any_resolved = false; + let gpu_ids: std::collections::BTreeSet<&str> = resolved + .into_iter() + .inspect(|_| any_resolved = true) + .filter_map(|(id, gpu)| gpu.then_some(id)) + .collect(); + if !any_resolved { + return None; + } + Some(GpuScenarioCounts { + resolved: gpu_ids.len(), + executed: executed + .into_iter() + .filter(|id| gpu_ids.contains(id)) + .count(), + }) +} + +/// Whether the run exits non-zero after reconciliation. +/// +/// Any one of a stale XPASS, an unexpected failure (both counted by the +/// harness) or a fatal post-run verdict is enough. Parse and hook errors exit +/// earlier, in the harness. +#[must_use] +pub const fn run_failed(stale_xpass: usize, unexpected_fail: usize, post_run: &Verdict) -> bool { + stale_xpass > 0 || unexpected_fail > 0 || post_run.fatal.is_some() +} + +/// A GitHub Actions workflow command (`::error::…`, `::notice::…`) carrying +/// `message` as its data. +/// +/// The data is escaped as the runner expects — `%` first, then CR and LF — so a +/// probe's stderr tail can neither cut the annotation short at a newline nor +/// have a literal `%0A` in it read back as one. +#[must_use] +pub fn workflow_command(command: &str, message: &str) -> String { + let data = message + .replace('%', "%25") + .replace('\r', "%0D") + .replace('\n', "%0A"); + format!("::{command}::{data}") +} + +/// One scenario the harness's filter resolved, as [`finish_run`] needs it. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedRun<'s> { + pub id: &'s str, + pub feature: &'s str, + pub scenario: &'s str, + pub engine: &'s str, + pub expectation: &'s crate::expectation::Expectation, + /// [`needs_gpu`] of the scenario's tags. + pub needs_gpu: bool, +} + +/// The post-run lane check, and the `platform.json` manifest that records its +/// failure as `lane_failure` so the consolidated report shows the lane red. +/// +/// `resolved` is every scenario the harness's filter saw (empty when a CLI +/// filter replaced it); `executed` is the id of each scenario in +/// `report.json`. +#[must_use] +pub fn finish_run<'a>( + cap: &'a crate::capability::HostCapability, + versions: crate::capability::PlatformVersions, + resolved: &[ResolvedRun<'_>], + executed: &[&str], + declaration: GpuDeclaration, +) -> (Verdict, crate::expectation::PlatformManifest<'a>) { + let gpu = gpu_scenario_counts( + resolved.iter().map(|r| (r.id, r.needs_gpu)), + executed.iter().copied(), + ); + let verdict = post_run_verdict(executed.len(), gpu, declaration); + let manifest = crate::expectation::PlatformManifest { + platform_slug: &cap.platform_slug, + capability: cap, + versions, + lane_failure: verdict.fatal.clone(), + expectations: resolved + .iter() + .map(|r| { + crate::expectation::ResolvedScenario::new( + r.id, + r.feature, + r.scenario, + r.engine, + r.expectation, + ) + }) + .collect(), + }; + (verdict, manifest) +} + +/// The lines the harness prints when a lane check fails: a GitHub `::error::` +/// annotation, then the same message as a plain line for anyone reading the +/// log outside Actions. +#[must_use] +pub fn failure_lines(message: &str) -> [String; 2] { + let line = format!("E2E lane check failed: {message}"); + [workflow_command("error", &line), line] +} + +/// The `::notice::` line the harness prints for a lane-check note. +#[must_use] +pub fn note_line(note: &str) -> String { + workflow_command("notice", &format!("E2E lane: {note}")) +} + +#[cfg(test)] +mod tests { + use super::{ + GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, failure_lines, + finish_run, gpu_scenario_counts, needs_gpu, note_line, parse_gpu_declaration, + post_run_verdict, preflight, preflight_verdict, run_failed, workflow_command, + }; + use std::ffi::OsStr; + + const FACTS: &str = + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask"; + const PROBE_ERROR: &str = "`rocm examine` exited with exit status: 1"; + + const ALL: [GpuDeclaration; 3] = [ + GpuDeclaration::Expected, + GpuDeclaration::NotExpected, + GpuDeclaration::Undeclared, + ]; + + #[test] + fn only_one_zero_and_unset_or_empty_are_declarations() { + assert_eq!(parse_gpu_declaration(None), Ok(GpuDeclaration::Undeclared)); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(""))), + Ok(GpuDeclaration::Undeclared) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("1"))), + Ok(GpuDeclaration::Expected) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("0"))), + Ok(GpuDeclaration::NotExpected) + ); + } + + #[test] + fn a_misspelt_declaration_is_an_error_not_undeclared() { + for value in ["true", "yes", " 1", "2"] { + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(value))), + Err(format!( + "invalid E2E_EXPECT_GPU value {value:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset" + )), + ); + } + } + + // Row: probe failed (non-zero exit, spawn failure, empty output, or + // `engines list` failing after `examine` succeeded all reach here as one + // `probe_error`). Fatal under every declaration, GPU or not. + #[test] + fn a_failed_probe_is_fatal_under_every_declaration() { + for declaration in ALL { + for has_amd_gpu in [false, true] { + let verdict = preflight_verdict(Some(PROBE_ERROR), has_amd_gpu, FACTS, declaration); + assert_eq!( + verdict, + Verdict { + fatal: Some(format!( + "the host capability probe could not read the rocm binary under \ + test, so every capability-gated scenario would resolve to skip and \ + the run would pass without testing them: {PROBE_ERROR}" + )), + notes: vec![], + }, + "{declaration:?}, gpu={has_amd_gpu}" + ); + } + } + } + + // Row: probe succeeded, no usable GPU, declared 1. + #[test] + fn a_declared_gpu_the_probe_did_not_find_is_fatal_and_names_the_facts() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::Expected); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but the capability probe found no \ + usable AMD GPU (detected_gfx_target=, wsl=false, driver_status=, \ + no visibility mask), so every @requires-gpu scenario would resolve to skip" + ) + ); + assert!(verdict.notes.is_empty()); + } + + // Row: probe succeeded with a GPU, declared 1 — runs, nothing to say. + #[test] + fn a_declared_gpu_that_is_present_runs_without_comment() { + assert_eq!( + preflight_verdict(None, true, FACTS, GpuDeclaration::Expected), + Verdict::default() + ); + } + + // Row: probe succeeded with a GPU, declared 0 — runs, with a stale note. + #[test] + fn a_declared_absent_gpu_that_is_present_is_a_note_not_a_failure() { + let verdict = preflight_verdict(None, true, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios do run on a host + // whose probe found a GPU, whatever the declaration, is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. + assert_eq!( + verdict.notes, + [format!( + "this lane declares no usable GPU (E2E_EXPECT_GPU=0) but the capability probe \ + found one ({FACTS}); the declaration is stale, and @requires-gpu scenarios will \ + run" + )] + ); + } + + // Row: probe succeeded, no GPU, declared 0 — runs, says why GPU scenarios + // are not applicable. + #[test] + fn a_declared_absent_gpu_that_is_absent_runs_with_a_note() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios resolve to skip + // (not applicable) where the probe found no GPU is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. + assert_eq!( + verdict.notes, + [ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are \ + not applicable here" + ] + ); + } + + // Rows: probe succeeded, undeclared — today's behaviour, GPU or not. + #[test] + fn an_undeclared_lane_runs_without_comment_either_way() { + for has_amd_gpu in [false, true] { + assert_eq!( + preflight_verdict(None, has_amd_gpu, FACTS, GpuDeclaration::Undeclared), + Verdict::default(), + "gpu={has_amd_gpu}" + ); + } + } + + // Post-run row: nothing executed — fatal under every declaration, whether + // or not the harness's own filter ran (a `-n` filter matching nothing is + // the `None` case). + #[test] + fn a_run_that_executed_nothing_is_fatal_under_every_declaration() { + let gpu_cases = [ + None, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + Some(GpuScenarioCounts { + resolved: 4, + executed: 0, + }), + ]; + for declaration in ALL { + for gpu in gpu_cases { + let verdict = post_run_verdict(0, gpu, declaration); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check \ + the cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and \ + E2E_ALSO_RUN_IDS" + ), + "{declaration:?}, {gpu:?}" + ); + } + } + } + + // Post-run row: declared 1, scenarios ran but no GPU one did. + #[test] + fn a_gpu_lane_on_which_no_gpu_scenario_ran_is_fatal() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (7 resolved, none ran)" + ) + ); + } + + // Post-run row: declared 1, scenarios ran, and the harness's filter ran but + // resolved no GPU scenario at all (none selected): still fatal, since the + // lane promised GPU coverage it did not get. + #[test] + fn a_gpu_lane_that_resolved_no_gpu_scenario_is_fatal() { + let verdict = post_run_verdict( + 5, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict, + Verdict { + fatal: Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (0 resolved, none ran)" + .to_owned() + ), + notes: vec![], + } + ); + } + + #[test] + fn a_gpu_lane_on_which_a_gpu_scenario_ran_passes() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 1, + }), + GpuDeclaration::Expected, + ), + Verdict::default() + ); + } + + // Post-run row: a CLI filter bypassed the harness's filter, so which + // scenarios need a GPU is unknown — only the zero check applies. + #[test] + fn a_cli_filtered_run_is_not_held_to_the_gpu_check() { + for declaration in ALL { + assert_eq!( + post_run_verdict(3, None, declaration), + Verdict::default(), + "{declaration:?}" + ); + } + } + + #[test] + fn a_no_gpu_lane_says_how_many_gpu_scenarios_did_not_run() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!(verdict.fatal, None); + assert_eq!( + verdict.notes, + [ + "7 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] + ); + // Some ran: the note counts only the ones that did not. + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 3, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!( + verdict.notes, + [ + "4 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] + ); + } + + #[test] + fn an_undeclared_lane_is_not_held_to_the_gpu_check() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Undeclared, + ), + Verdict::default() + ); + } + + // --- the harness glue, as pure functions --- + + #[test] + fn preflight_refuses_an_invalid_declaration_before_reading_the_probe() { + let invalid = Err( + "invalid E2E_EXPECT_GPU value \"yes\": use 1 (this lane has a GPU), \ + 0 (it deliberately has none), or leave it unset" + .to_owned(), + ); + // A probe that also failed, or found no GPU, must not displace the + // declaration error: the declaration is checked first. + for (probe_error, has_amd_gpu) in [(None, true), (Some(PROBE_ERROR), false), (None, false)] + { + assert_eq!( + preflight(Some(OsStr::new("yes")), probe_error, has_amd_gpu, FACTS), + invalid, + "probe_error={probe_error:?}, gpu={has_amd_gpu}" + ); + } + } + + #[test] + fn preflight_refuses_a_failed_probe_and_a_missing_declared_gpu() { + let refused = preflight(None, Some(PROBE_ERROR), true, FACTS).expect_err("probe failed"); + assert!(refused.ends_with(PROBE_ERROR), "{refused}"); + let refused = + preflight(Some(OsStr::new("1")), None, false, FACTS).expect_err("declared GPU missing"); + assert!( + refused.starts_with("this lane declares a GPU (E2E_EXPECT_GPU=1)"), + "{refused}" + ); + } + + #[test] + fn preflight_carries_the_declaration_and_notes_into_the_run() { + assert_eq!( + preflight(Some(OsStr::new("0")), None, false, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::NotExpected, + notes: vec![ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu \ + scenarios are not applicable here" + .to_owned() + ], + }) + ); + assert_eq!( + preflight(Some(OsStr::new("1")), None, true, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::Expected, + notes: vec![], + }) + ); + } + + #[test] + fn gpu_and_multi_gpu_scenarios_need_a_gpu_and_a_gfx_target_one_does_not() { + let decl = crate::expectation::ScenarioDecl::from_tags; + assert!(needs_gpu(&decl(&["id:a", "requires-gpu"]))); + assert!(needs_gpu(&decl(&["id:b", "requires-multi-gpu"]))); + assert!(!needs_gpu(&decl(&["id:c", "requires-gfx-target"]))); + assert!(!needs_gpu(&decl(&["id:d"]))); + } + + #[test] + fn a_cli_filtered_run_resolved_nothing_so_has_no_gpu_counts() { + assert_eq!(gpu_scenario_counts([], ["serve-a", "serve-b"]), None); + } + + #[test] + fn gpu_counts_take_resolved_from_the_filter_and_executed_from_the_report() { + let resolved = [ + ("gpu-ran", true), + ("multi-gpu-only", true), + ("gpu-skipped", true), + ("cpu-ran", false), + ]; + // `cpu-ran` executed but is not a GPU scenario, and `untagged` was + // never resolved: neither counts as an executed GPU scenario. + let executed = ["gpu-ran", "multi-gpu-only", "cpu-ran", "untagged"]; + assert_eq!( + gpu_scenario_counts(resolved, executed), + Some(GpuScenarioCounts { + resolved: 3, + executed: 2, + }) + ); + } + + #[test] + fn gpu_counts_with_nothing_executed_are_zero_not_none() { + assert_eq!( + gpu_scenario_counts([("gpu", true), ("cpu", false)], []), + Some(GpuScenarioCounts { + resolved: 1, + executed: 0, + }) + ); + assert_eq!( + gpu_scenario_counts([("cpu", false)], ["cpu"]), + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }) + ); + } + + #[test] + fn the_run_fails_on_any_one_of_its_three_reasons() { + let fatal = Verdict { + fatal: Some("0 scenarios executed".to_owned()), + notes: vec![], + }; + let clean = Verdict::default(); + assert!(!run_failed(0, 0, &clean)); + assert!(run_failed(1, 0, &clean)); + assert!(run_failed(0, 1, &clean)); + assert!(run_failed(0, 0, &fatal)); + // A note alone never fails the run. + let noted = Verdict { + fatal: None, + notes: vec!["stale".to_owned()], + }; + assert!(!run_failed(0, 0, ¬ed)); + } + + #[test] + fn workflow_command_data_escapes_percent_cr_and_lf() { + assert_eq!( + workflow_command("error", "100% failed\r\nstderr: a%0Ab"), + "::error::100%25 failed%0D%0Astderr: a%250Ab" + ); + assert_eq!(workflow_command("notice", "plain"), "::notice::plain"); + } + + #[test] + fn a_failure_prints_an_escaped_annotation_then_the_plain_message() { + assert_eq!( + failure_lines("probe failed\n--- stderr ---\n100% full"), + [ + "::error::E2E lane check failed: probe failed%0A--- stderr ---%0A100%25 full" + .to_owned(), + "E2E lane check failed: probe failed\n--- stderr ---\n100% full".to_owned(), + ] + ); + } + + #[test] + fn a_note_prints_as_an_escaped_notice() { + assert_eq!( + note_line("7 of 7 GPU scenario(s) did not run (100%)"), + "::notice::E2E lane: 7 of 7 GPU scenario(s) did not run (100%25)" + ); + } + + /// `finish_run`'s manifest, written as the harness writes it and read back + /// with the consolidated report's own loader: the post-run lane failure + /// it records makes the lane red there, and a lane that passed the check + /// stays green. Ties the producer to the consumer through the function + /// `tests/e2e.rs` calls. + #[test] + fn finish_runs_manifest_reads_back_as_the_verdict_in_the_report() { + let cap = crate::capability::HostCapability { + os_family: "linux".into(), + is_wsl: false, + gfx_target: Some("gfx1151".into()), + has_amd_gpu: true, + amd_gpu_count: Some(1), + available_engines: vec!["lemonade".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "strix-halo-linux".into(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, + }; + let pass = crate::expectation::Expectation::ExpectPass; + let skip = crate::expectation::Expectation::Skip { + reason: "no usable AMD GPU".to_owned(), + }; + let resolved_with = |gpu_expectation| { + [ + ResolvedRun { + id: "examine-version", + feature: "Examine", + scenario: "examine-01", + engine: "lemonade", + expectation: &pass, + needs_gpu: false, + }, + ResolvedRun { + id: "serve-gpu", + feature: "Serving", + scenario: "serve-01", + engine: "vllm", + expectation: gpu_expectation, + needs_gpu: true, + }, + ] + }; + // Writes what the harness writes and reads it back: the verdict, the + // report's status, and `platform.json`'s id -> expected mapping. + // `executed` is also what report.json holds; every listed id passed. + let status_of = |resolved: &[ResolvedRun<'_>], executed: &[&str]| { + let (verdict, manifest) = finish_run( + &cap, + crate::capability::PlatformVersions::default(), + resolved, + executed, + GpuDeclaration::Expected, + ); + let dir = tempfile::TempDir::with_prefix("rocm-e2e-manifest-").expect("temp dir"); + let elements: Vec = executed + .iter() + .map(|id| { + format!( + r#"{{"name":"{id}","tags":[{{"name":"id:{id}"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"passed"}}}}]}}"# + ) + }) + .collect(); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{}]}}]"#, + elements.join(",") + ), + ) + .expect("report.json"); + let platform = dir.path().join("platform.json"); + std::fs::write( + &platform, + serde_json::to_string_pretty(&manifest).expect("serializes"), + ) + .expect("platform.json"); + let written: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&platform).expect("read back")) + .expect("valid json"); + let mapping: Vec> = written["expectations"] + .as_array() + .expect("an expectations array") + .iter() + .map(|e| { + ["id", "feature", "scenario", "effective_engine", "expected"] + .iter() + .map(|key| e[*key].as_str().unwrap_or_default().to_owned()) + .collect() + }) + .collect(); + ( + verdict, + e2e_report::platform_status("e2e-report", &report), + mapping, + ) + }; + // id, feature, scenario, effective_engine, expected — per expectation. + let mapping_of = |rows: &[[&str; 5]]| -> Vec> { + rows.iter() + .map(|row| row.iter().map(|field| (*field).to_owned()).collect()) + .collect() + }; + + // Every cell is pass or skip, so only the recorded lane failure can + // make this row FAIL. + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; + let (verdict, failed, mapping) = status_of(&resolved_with(&skip), &["examine-version"]); + assert_eq!(verdict.fatal.as_deref(), Some(reason)); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "skip"], + ]) + ); + assert_eq!( + failed, + e2e_report::PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + + let (verdict, passed, mapping) = + status_of(&resolved_with(&pass), &["examine-version", "serve-gpu"]); + assert_eq!(verdict.fatal, None); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "pass"], + ]) + ); + assert_eq!( + passed, + e2e_report::PlatformStatus { + ok: true, + status: "PASS", + lane_failure: None, + } + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 29e3345836..1d10fe54bb 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -12,6 +12,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lane_verdict; pub mod lemonade_backend; pub mod lemonade_diagnostics; pub mod lemonade_protocol; diff --git a/tests/e2e-cucumber/src/loopback_http.rs b/tests/e2e-cucumber/src/loopback_http.rs index 1c5736caa4..f962f45f4e 100644 --- a/tests/e2e-cucumber/src/loopback_http.rs +++ b/tests/e2e-cucumber/src/loopback_http.rs @@ -30,8 +30,11 @@ use tower_http::services::ServeDir; use crate::http_server::{self, ServerHandle}; -/// A minimal loopback HTTP file server serving one directory, for the Windows -/// native-HTTP install scenario. Shuts down on drop. +/// A minimal loopback HTTP file server serving one directory. +/// +/// Backs the install lifecycle scenarios: the Windows native-HTTP install, and +/// the Linux progress-rendering runs, which need a real HTTP download to draw a +/// bar for. Shuts down on drop. #[derive(Debug)] pub struct LoopbackServer { server: ServerHandle, diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 34b0ab36aa..a50c1b5c8f 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -374,6 +374,67 @@ struct ServerState { /// `Some` only when started via [`MockServer::start_with_chat_script`]; the /// replies to hand out, in request order. `None` keeps the fixed stub reply. chat_script: Option>>, + /// `Some` only when started via [`MockServer::start_with_tool_call`]; makes + /// the chat route answer one user turn with a tool call instead of text. + tool_call: Option>, +} + +/// A tool call the mock model makes in reply to a user turn containing +/// `trigger` (case-insensitive). Once the client answers with the tool's +/// result, the mock goes back to its canned text reply, so the client's tool +/// loop ends after one round. +/// +/// Keyed on the user's words rather than on the request's position (as +/// [`MockServer::start_with_chat_script`] is) because an interactive chat +/// sends other completions first — the local-endpoint detection probe — and +/// their number is not this harness's to fix. +#[derive(Debug)] +struct ScriptedToolCall { + trigger: String, + name: String, + arguments: Value, +} + +impl ScriptedToolCall { + /// Whether `body`'s newest message is a user turn asking for this call. + fn fires_for(&self, body: &Value) -> bool { + let Some(last) = body + .get("messages") + .and_then(Value::as_array) + .and_then(|messages| messages.last()) + else { + return false; + }; + last.get("role").and_then(Value::as_str) == Some("user") + && chat_message_text(last.get("content").unwrap_or(&Value::Null)) + .to_lowercase() + .contains(&self.trigger) + } +} + +/// The text of one OpenAI-format message `content`. +/// +/// A bare string, or an array of typed parts whose `text` fields are joined. +/// `content` is a bare string on the turns the TUI builds, but an array of +/// typed parts on the system message the chat client emits — read both, or +/// text looks absent when it is simply wrapped. +#[must_use] +pub fn chat_message_text(content: &Value) -> String { + content.as_str().map_or_else( + || { + content + .as_array() + .map(|parts| { + parts + .iter() + .filter_map(|p| p.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n") + }) + .unwrap_or_default() + }, + str::to_owned, + ) } /// Deterministic, monotonically-advancing state behind the mock `/metrics` @@ -478,6 +539,7 @@ impl MockServer { chat_paths: Arc::clone(&chat_paths), completion_requests: Arc::clone(&completion_requests), chat_script: None, + tool_call: None, }; let app = Router::new() .route("/v1/models", get(handle_models)) @@ -522,6 +584,7 @@ impl MockServer { chat_paths: Arc::clone(&chat_paths), completion_requests: Arc::clone(&completion_requests), chat_script: None, + tool_call: None, }; let app = Router::new() .route("/v1/models", get(handle_models)) @@ -565,7 +628,38 @@ impl MockServer { .failure_count() } + /// Like [`Self::start`], but a user turn containing `trigger` + /// (case-insensitive) is answered with one call to the tool `name` with + /// `arguments`, as a model using the client's tools would. The client's + /// follow-up request carries the tool's result; it gets the canned text + /// reply, so the scenario can assert on what the client reported back. + pub async fn start_with_tool_call( + model_name: &str, + trigger: &str, + name: &str, + arguments: Value, + ) -> Self { + Self::spawn_with( + model_name, + false, + Some(Arc::new(ScriptedToolCall { + trigger: trigger.to_lowercase(), + name: name.to_string(), + arguments, + })), + ) + .await + } + async fn spawn(model_name: &str, with_metrics: bool) -> Self { + Self::spawn_with(model_name, with_metrics, None).await + } + + async fn spawn_with( + model_name: &str, + with_metrics: bool, + tool_call: Option>, + ) -> Self { let last_chat_request = Arc::new(Mutex::new(None)); let chat_paths = Arc::new(Mutex::new(Vec::new())); let completion_requests = Arc::new(Mutex::new(Vec::new())); @@ -577,6 +671,7 @@ impl MockServer { chat_paths: Arc::clone(&chat_paths), completion_requests: Arc::clone(&completion_requests), chat_script: None, + tool_call, }; let mut app = Router::new() @@ -624,6 +719,7 @@ impl MockServer { chat_paths: Arc::clone(&chat_paths), completion_requests: Arc::clone(&completion_requests), chat_script: Some(Arc::new(script)), + tool_call: None, }; let app = Router::new() .route("/v1/models", get(handle_models)) @@ -1021,6 +1117,12 @@ async fn handle_chat( .unwrap_or("") .to_string(); + let tool_call_reply = state + .tool_call + .as_ref() + .filter(|tool_call| tool_call.fires_for(&body)) + .map(|tool_call| chat_tool_call_reply(&model, &tool_call.name, &tool_call.arguments)); + // Record the request body so scenarios can assert on exactly what the CLI // sent (see `MockServer::last_chat_request` / `wait_for_chat_request`) — // the canned reply below never varies with the prompt, so without this a @@ -1061,6 +1163,10 @@ async fn handle_chat( ); } + if let Some(reply) = tool_call_reply { + return (StatusCode::OK, Json(reply)); + } + // Tests assert on the request contract, not the reply text, so the default // is a fixed stub. Demos (`rocm-demo-env`) set `ROCM_MOCK_CHAT_REPLY` to a // realistic answer so the recorded screencast reads naturally. @@ -1239,6 +1345,60 @@ mod chat_script_tests { assert_eq!(arguments["request"], "serve a model"); } + #[tokio::test] + async fn a_scripted_tool_call_answers_only_the_triggering_user_turn() { + let mock = MockServer::start_with_tool_call( + "m", + "Start A Server", + "launch_server", + json!({"model": "shown/model"}), + ) + .await; + + // An unrelated turn gets the canned text, not the tool call. + let (_, body) = post_chat(&mock, "hello").await; + assert!(body["choices"][0]["message"]["tool_calls"].is_null()); + + // The trigger matches case-insensitively, on both sides: the trigger was + // given mixed-case above and the user's words are mixed-case here. + let (_, body) = post_chat(&mock, "Please START a server now").await; + let call = &body["choices"][0]["message"]["tool_calls"][0]["function"]; + assert_eq!(call["name"], "launch_server"); + let arguments: Value = + serde_json::from_str(call["arguments"].as_str().expect("arguments not a string")) + .expect("arguments not JSON"); + assert_eq!(arguments, json!({"model": "shown/model"})); + + // The follow-up carrying the tool's result ends the loop with text, + // even though the triggering user turn is still in the history. Only a + // user turn fires the call, so a tool message repeating the trigger + // words does not. + let response = client() + .post(format!("{}/chat/completions", mock.base_url())) + .json(&json!({ + "model": "m", + "messages": [ + {"role": "user", "content": "Please START a server now"}, + {"role": "tool", "content": "refused: start a server"} + ] + })) + .send() + .await + .expect("chat request failed"); + let body: Value = response.json().await.expect("non-JSON chat reply"); + assert!(body["choices"][0]["message"]["tool_calls"].is_null()); + assert!(!reply_text(&body).is_empty()); + + // Every request is recorded, the one answered with the tool call + // included: a step reads the CLI's follow-up out of these. + let recorded = mock.chat_requests(); + assert_eq!(recorded.len(), 3, "{recorded:?}"); + assert_eq!( + recorded[1]["messages"][0]["content"], + "Please START a server now" + ); + } + #[tokio::test] async fn without_a_script_every_request_gets_the_same_stub_and_is_recorded() { let mock = MockServer::start("m").await; diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6e8db6c579..6174cab2a6 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -39,6 +39,7 @@ mod e2e { pub mod planner_steps; pub mod remote_steps; pub mod runtime_lifecycle_steps; + pub mod runtime_side_by_side_steps; pub mod runtime_steps; pub mod service_cleanup_steps; pub mod service_stop_steps; @@ -51,6 +52,7 @@ mod e2e { pub mod tui_driver; pub mod uninstall_steps; pub mod update_steps; + pub mod vllm_tensorizer_steps; } // ── World ────────────────────────────────────────────────────────── @@ -171,6 +173,9 @@ pub struct E2eWorld { /// from. Consumed by [`run_rocm_with_scenario_env`] like `command_env`; `None` /// leaves the harness's own working directory. pub command_dir: Option, + /// `(request, HTTP status)` for each server route a step called directly, + /// in call order, for a later Then step to judge. + pub route_statuses: Vec<(String, u16)>, /// The interactive dash/chat TUI spawned under a pseudo-terminal for this /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the /// mock server and isolated directory so the child process never outlives @@ -195,6 +200,11 @@ pub struct E2eWorld { /// dir, captured logs). `Some` only for `@lifecycle` scenarios; all its paths /// are rooted in `isolated_root` so teardown removes them with the temp dir. pub lifecycle: Option, + /// The runtimes a side-by-side scenario planted, with the bytes of their + /// install records as planted and what each activation left behind, so a + /// later step can tell an untouched record from a rewritten one. `Some` + /// only for the side-by-side scenarios in `runtime_lifecycle.feature`. + pub side_by_side: Option, /// Raw text of `skills/rocm-doctor/reference.md`, loaded by the rocm-doctor /// skill scenarios. That document is the EXPECTED-value fixture for the /// skill↔CLI contract checks — see `e2e::skill_steps`. @@ -240,6 +250,9 @@ pub struct E2eWorld { /// local webhook it listens on. Held here so `Drop` can kill and reap it /// whatever the scenario's outcome. pub automation_daemon: Option, + /// Every file of a Lemonade tree a Given planted, with its bytes, so a Then + /// can show a refused install left that tree exactly as it was. + pub planted_tree_snapshot: Option)>>, } /// One scenario's resolved expectation plus the identity needed to report it. @@ -253,6 +266,9 @@ struct Resolution { engine: String, feature: String, scenario: String, + /// Tagged `@requires-gpu` or `@requires-multi-gpu`, for the post-run check + /// that a GPU-declaring lane executed at least one of them. + requires_gpu: bool, } /// Resolve a CI-provided shared-directory env var to a validated, existing path. @@ -395,11 +411,13 @@ impl Default for E2eWorld { expect_xfail: false, command_env: Vec::new(), command_dir: None, + route_statuses: Vec::new(), tui: None, chat_use_mock: false, comfyui_baseline_torch_stack: None, comfyui_baseline_distributions: None, lifecycle: None, + side_by_side: None, skill_reference: None, skill_symptom: None, symptom_paste: None, @@ -409,6 +427,7 @@ impl Default for E2eWorld { recorded_descendant: None, dash_report_refused: None, automation_daemon: None, + planted_tree_snapshot: None, } } } @@ -973,6 +992,18 @@ pub fn run_rocm_bounded( world: &E2eWorld, args: &[&str], timeout: std::time::Duration, +) -> (String, String, i32) { + run_rocm_bounded_with_env(world, args, &[], timeout) +} + +/// Like [`run_rocm_bounded`], but sets `envs` on the child on top of the +/// scenario's isolated env, for a value that must not appear in the recorded +/// argv (an endpoint API key). +pub fn run_rocm_bounded_with_env( + world: &E2eWorld, + args: &[&str], + envs: &[(&str, &str)], + timeout: std::time::Duration, ) -> (String, String, i32) { use std::process::Stdio; @@ -983,6 +1014,7 @@ pub fn run_rocm_bounded( let mut cmd = std::process::Command::new(&binary); cmd.args(args); world.isolate_cmd(&mut cmd); + cmd.envs(envs.iter().copied()); cmd.stdin(Stdio::null()); cmd.stdout(Stdio::from( std::fs::File::create(&out_path).expect("stdout file"), @@ -1176,6 +1208,37 @@ pub fn run_rocm_without_data_dir_override( /// Run `rocm` with the behavioral fixture established by a Given step, then /// consume it so it cannot leak into a later action in the same scenario. pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { + let (mut cmd, binary) = scenario_command(world, args); + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + scenario_command_result(world, args, &output) +} + +/// [`run_rocm_with_scenario_env`] for a run that may take long, waited for on +/// a blocking thread instead of the one the whole suite is polled on. +/// +/// The cucumber run is one future under `block_on` (see `main`), so a +/// synchronous wait inside a step stops every other concurrent scenario for as +/// long as it lasts. A fix that waits out its one-minute timeout (diagnose-68) +/// stalled the TUI scenarios running beside it past their own wall-clock caps +/// ("the harness left this step unpolled"). The same reason +/// `probe_device_memory` in `serving_steps.rs` runs on `spawn_blocking`. +pub async fn run_rocm_with_scenario_env_off_executor( + world: &mut E2eWorld, + args: &[&str], +) -> (String, String, i32) { + let (mut cmd, binary) = scenario_command(world, args); + let output = tokio::task::spawn_blocking(move || cmd.output()) + .await + .unwrap_or_else(|e| panic!("the thread running {binary} failed: {e}")) + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + scenario_command_result(world, args, &output) +} + +/// The `rocm` command [`run_rocm_with_scenario_env`] runs, with the scenario's +/// fixture moved into it, and the binary's name for messages. +fn scenario_command(world: &mut E2eWorld, args: &[&str]) -> (std::process::Command, String) { let binary = rocm_binary(); // A binary given as a relative path was meant relative to this process's // directory, not to the scenario's, so it is made absolute before the child @@ -1197,9 +1260,16 @@ pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (Strin if let Some(dir) = world.command_dir.take() { cmd.current_dir(dir); } - let output = cmd - .output() - .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + (cmd, binary) +} + +/// `(stdout, stderr, rc)` of a finished scenario command, recorded for the +/// command-coverage report. +fn scenario_command_result( + world: &E2eWorld, + args: &[&str], + output: &std::process::Output, +) -> (String, String, i32) { let rc = output.status.code().unwrap_or(-1); let stdout = String::from_utf8_lossy(&output.stdout).to_string(); record_command(world.current_scenario.as_deref(), args, rc, &stdout); @@ -1449,6 +1519,7 @@ async fn run_suite() { use e2e_cucumber::expectation::{ Expectation, ScenarioDecl, lifecycle_lane_includes, resolve, unmatched_also_run_ids, }; + use e2e_cucumber::lane_verdict; use e2e_cucumber::monotonic_clock::MonotonicClockWriter; let dir = results_dir(); @@ -1509,6 +1580,19 @@ async fn run_suite() { cap.platform_slug, cap.os_family, cap.has_amd_gpu, cap.effective_serve_engine, ); + // Refuse to run when the result could not mean anything: a probe that could + // not read the binary, or a lane that declares a GPU the probe did not find, + // would resolve every GPU scenario to skip and pass. See `lane_verdict`. + let preflight = lane_verdict::preflight( + std::env::var_os(lane_verdict::EXPECT_GPU_ENV).as_deref(), + cap.probe_error.as_deref(), + cap.has_amd_gpu, + &e2e_cucumber::capability::probe_facts_summary(cap), + ) + .unwrap_or_else(|fatal| lane_failure(&fatal)); + print_lane_notes(&preflight.notes); + let gpu_declaration = preflight.declaration; + // Shared record of each scenario's resolved expectation, keyed by @id. // Populated by `filter_run` (which sees every scenario, run or skipped) so // the post-run evaluation and platform.json can reconcile by id — including @@ -1631,6 +1715,7 @@ async fn run_suite() { engine, feature: feature.name.clone(), scenario: scenario.name.clone(), + requires_gpu: lane_verdict::needs_gpu(&decl), }, ); // Two scenarios sharing an `@id` would silently overwrite each @@ -1686,31 +1771,41 @@ async fn run_suite() { // OS is always collected (every platform, incl. mock); ROCm/vLLM/lemonade come // from the shared runtimes tree CI provides (E2E_SHARED_RUNTIMES_DIR) — absent // on mock (no runtime) and on local runs (per-scenario dirs already dropped), - // where those render as "n/a". Best-effort. + // where those render as "n/a". Best-effort. Collected before the lane check + // so that, once decided, nothing stands between it and platform.json. let shared = shared_runtimes_dir(); let versions = e2e_cucumber::capability::collect_versions(shared.as_deref()); + // A run with nothing to fail passes; check it executed something, and on + // a GPU-declaring lane something that needed the GPU. `resolutions` is empty + // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then + // which scenarios need a GPU is unknown. The verdict goes into platform.json, + // so the consolidated report shows a failed lane check as red. + let resolved: Vec> = resolutions + .iter() + .map(|(id, r)| lane_verdict::ResolvedRun { + id, + feature: &r.feature, + scenario: &r.scenario, + engine: &r.engine, + expectation: &r.expectation, + needs_gpu: r.requires_gpu, + }) + .collect(); + let executed: Vec<&str> = actual.iter().map(|(id, _)| id.as_str()).collect(); + let (post_run, manifest) = + lane_verdict::finish_run(cap, versions, &resolved, &executed, gpu_declaration); + print_lane_notes(&post_run.notes); + // Write the platform.json sidecar (probed capability + every resolution, - // including skips) for the central report's expected-vs-actual grid. - let manifest = e2e_cucumber::expectation::PlatformManifest { - platform_slug: &cap.platform_slug, - capability: cap, - versions, - expectations: resolutions - .iter() - .map(|(id, r)| { - e2e_cucumber::expectation::ResolvedScenario::new( - id, - &r.feature, - &r.scenario, - &r.engine, - &r.expectation, - ) - }) - .collect(), - }; - if let Ok(json) = serde_json::to_string_pretty(&manifest) { - let _ = std::fs::write(dir.join("platform.json"), json); + // including skips, + any lane failure) for the central report's grid. + let written = serde_json::to_string_pretty(&manifest) + .map_err(|error| error.to_string()) + .and_then(|json| { + std::fs::write(dir.join("platform.json"), json).map_err(|error| error.to_string()) + }); + if let Err(error) = written { + eprintln!("warning: could not write platform.json: {error}"); } // Classify actual results against the matrix. A deterministic XPASS is @@ -1755,7 +1850,7 @@ async fn run_suite() { "XPASS (flaky, tolerated): '{x}' passed this run; the intermittent bug remains tracked." ); } - if !stale_xpass.is_empty() || !unexpected_fail.is_empty() { + if lane_verdict::run_failed(stale_xpass.len(), unexpected_fail.len(), &post_run) { for x in &stale_xpass { eprintln!( "XPASS: '{x}' is expected to fail on this host but PASSED \u{2014} the bug appears \ @@ -1767,6 +1862,24 @@ async fn run_suite() { "FAIL: '{f}' was expected to pass on this host but FAILED \u{2014} a regression." ); } + if let Some(fatal) = &post_run.fatal { + lane_failure(fatal); + } std::process::exit(1); } } + +/// Print a lane-check failure (see `lane_verdict::failure_lines`) and exit +/// non-zero. +fn lane_failure(message: &str) -> ! { + for line in e2e_cucumber::lane_verdict::failure_lines(message) { + eprintln!("{line}"); + } + std::process::exit(1); +} + +fn print_lane_notes(notes: &[String]) { + for note in notes { + eprintln!("{}", e2e_cucumber::lane_verdict::note_line(note)); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index ed401a4306..dbfeda1d0f 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -200,3 +200,101 @@ async fn assert_model_generated_reply(world: &mut E2eWorld) { .unwrap_or(""); assert!(!content.is_empty(), "empty reply in chat response: {resp}"); } + +// ── Direct tool calls through `rocm mcp-call` ────────────────────── + +/// A directory that exists wherever the suite runs: this crate's own. +const EXISTING_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +fn missing_path() -> String { + std::path::Path::new(EXISTING_DIR) + .join("e2e-no-such-path") + .display() + .to_string() +} + +fn run_path_exists(world: &mut E2eWorld, arguments: &serde_json::Value) { + let arguments = arguments.to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["mcp-call", "path_exists", "--arguments-json", &arguments], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("a tool-using client hands the CLI a whole tool-call request to check a path that exists")] +async fn mcp_call_with_a_params_envelope(world: &mut E2eWorld) { + // An MCP `tools/call` request's params, as a client received them: the + // tool's name, its arguments, and the protocol's optional `_meta`. + run_path_exists( + world, + &serde_json::json!({ + "name": "path_exists", + "arguments": { "path": EXISTING_DIR }, + "_meta": { "progressToken": 1 }, + }), + ); +} + +#[when( + "a tool-using client asks the CLI to check a path with a second set of arguments nested inside the first" +)] +async fn mcp_call_with_nested_arguments(world: &mut E2eWorld) { + // The outer path exists and the nested one does not, so a result for + // either would show which set ran. + run_path_exists( + world, + &serde_json::json!({ + "path": EXISTING_DIR, + "arguments": { "path": missing_path() }, + }), + ); +} + +#[then("the tool reports that path as an existing directory")] +async fn path_reported_as_existing_directory(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "rocm mcp-call failed:\nstdout: {stdout}\nstderr: {stderr}" + ); + let value: serde_json::Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("rocm mcp-call printed no JSON tool result ({e}):\n{stdout}")); + let text = value["content"][0]["text"].as_str().unwrap_or_default(); + // The tool's own report, for the path inside the request's arguments. + assert!( + text.contains(&format!("path: {EXISTING_DIR}\n")) + && text.contains("exists: true") + && text.contains("kind: directory"), + "the tool did not report the requested directory:\n{text}" + ); +} + +#[then("the CLI refuses the request for its nested arguments")] +async fn mcp_call_refused_for_nested_arguments(world: &mut E2eWorld) { + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert_ne!( + world.cli_rc, + Some(0), + "the request was not refused:\n{stderr}" + ); + assert!( + stderr.contains("must not contain a nested `arguments` field"), + "the refusal does not name the nested arguments field:\n{stderr}" + ); +} + +#[then("no tool result is printed")] +async fn no_tool_result_printed(world: &mut E2eWorld) { + // The tool prints its result on stdout only when it ran; neither path's + // report may appear. + let stdout = world.cli_output.as_deref().unwrap_or_default(); + assert!( + stdout.trim().is_empty(), + "a tool result was printed for a refused request:\n{stdout}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index c0bca4e7a5..24574e9294 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -1067,7 +1067,7 @@ async fn used_comfyui_install(world: &mut E2eWorld) { /// `tests/e2e/tui_driver.rs` (through `spawn_binary_with_env`, which overlays /// `command_env` onto the child and leaves it in place). It does not reach /// plain `run_rocm` or `run_rocm_ok`, which ignore `command_env`. -fn disable_startup_update_check(world: &mut E2eWorld) { +pub(crate) fn disable_startup_update_check(world: &mut E2eWorld) { world .command_env .push(("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1".into())); diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index ddb5b598d9..66696d3f35 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -8,7 +8,7 @@ //! screen, and clean exit — which the piped-`Command` steps structurally cannot. use cucumber::{given, then, when}; -use e2e_cucumber::mock_server::{MetricsMode, MockServer, ServiceRecordOptions}; +use e2e_cucumber::mock_server::{MetricsMode, MockServer, ServiceRecordOptions, chat_message_text}; use std::time::{Duration, Instant}; use crate::E2eWorld; @@ -17,6 +17,9 @@ use crate::e2e::tui_driver::{QUIT_CONFIRM_MARKER, TermSignal, TuiSession, defaul /// corresponding `Then` step (`managed_chat_request_carried_prompt`) asserts /// the mock actually received — so the two can never silently drift apart. const MANAGED_MODEL_PROMPT: &str = "hello from the terminal"; +/// What the user types to ask the managed model to start a server, and the +/// trigger the scripted mock answers with a `launch_server` tool call. +const START_SERVER_PROMPT: &str = "please start a model server"; /// The services overlay's own panel title, drawn by `draw_services_manager` on /// the overlay's border row. It is on screen exactly while the overlay is, so @@ -475,6 +478,30 @@ async fn running_managed_model(world: &mut E2eWorld) { setup_managed_model(world, ServiceRecordOptions::default(), false).await; } +#[given( + "a running managed model answers a request to start a server with a tool call carrying two sets of arguments" +)] +async fn managed_model_with_two_argument_sets(world: &mut E2eWorld) { + // The outer arguments are what an approval card built from the call shows; + // the nested ones name a different model and port. Which one a card or a + // replay reads is the whole question, so they differ in every field. + let model = "TestModel/E2E-1B"; + let mock = MockServer::start_with_tool_call( + model, + START_SERVER_PROMPT, + "launch_server", + serde_json::json!({ + "model": "TestModel/Shown-1B", + "arguments": { "model": "TestModel/Other-1B", "port": 8123 }, + }), + ) + .await; + world.endpoint = Some(mock.base_url()); + world.model_name = Some(model.to_string()); + world.mock = Some(mock); + world.register_mock_service_with(ServiceRecordOptions::default()); +} + /// Re-point "e2e-mock"'s `supervisor_pid`/`engine_pid` at a disposable `sleep` /// child instead of this harness's own PID. `register_mock_service_with` /// always points those fields at the harness process, which is fine for a @@ -1612,6 +1639,19 @@ async fn send_managed_model_message(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("failed to submit the chat message: {e}")); } +#[when("the user asks the managed model to start a model server")] +async fn ask_managed_model_to_start_server(world: &mut E2eWorld) { + let tui = session(world); + tui.wait_for_screen("No messages yet.", default_timeout()) + .await + .unwrap_or_else(|e| panic!("chat surface never became ready: {e}")); + // Consent already focused the input; see `send_managed_model_message`. + tui.send(START_SERVER_PROMPT) + .unwrap_or_else(|e| panic!("failed to type the chat message: {e}")); + tui.send("\r") + .unwrap_or_else(|e| panic!("failed to submit the chat message: {e}")); +} + #[when("the user sends a message about GPU health")] async fn send_gpu_message(world: &mut E2eWorld) { let tui = session(world); @@ -2388,6 +2428,61 @@ async fn tool_call_shown_declined(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("the declined-tool-call message did not appear: {e}")); } +#[then("the model is told the tool call was refused")] +async fn model_told_tool_call_refused(world: &mut E2eWorld) { + // The client reports a tool's outcome back to the model in a `tool` + // message on its next request. That message is the CLI's own account of + // what it did with the call, so it is asserted here, not the canned reply. + let body = world + .mock + .as_ref() + .expect("no mock server running") + .wait_for_chat_request_where(default_timeout(), |body| { + body.get("messages") + .and_then(serde_json::Value::as_array) + .is_some_and(|messages| { + messages + .iter() + .any(|m| m.get("role").and_then(serde_json::Value::as_str) == Some("tool")) + }) + }) + .await + .unwrap_or_else(|e| panic!("the model was never sent the tool call's outcome: {e}")); + let outcome = body["messages"] + .as_array() + .into_iter() + .flatten() + .filter(|m| m.get("role").and_then(serde_json::Value::as_str) == Some("tool")) + .map(|m| chat_message_text(m.get("content").unwrap_or(&serde_json::Value::Null))) + .collect::>() + .join("\n"); + assert!( + outcome.contains("nested `arguments`") && !outcome.contains("surfaced_for_approval"), + "the tool call should have been refused for its nested arguments, \ + not surfaced for approval; the model was told:\n{outcome}" + ); +} + +#[then("no tool approval prompt is displayed")] +async fn no_approval_prompt_displayed(world: &mut E2eWorld) { + // The model's reply to the refusal is the last thing this turn renders, so + // once it is on screen any approval prompt the call raised would be too. + // The dialog's title prefix and approve button are the strings the dash + // draws in `rocm_dash_tui::ui::approval` (`draw_approval_inner`, + // `buttons_line`); its test + // `an_approval_dialog_shows_its_review_title_and_approve_button` pins them + // there, so a rename fails that test instead of passing this check. + let tui = session(world); + tui.wait_for_screen("mock response for testing", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the model's reply to the refusal did not appear: {e}")); + let screen = tui.screen_text(); + assert!( + !screen.contains("Approve (y)") && !screen.contains("Review: "), + "an approval prompt was shown for a refused tool call:\n{screen}" + ); +} + #[then("the managed model's response is displayed")] async fn managed_model_response_displayed(world: &mut E2eWorld) { session(world) @@ -2452,8 +2547,10 @@ async fn recorded_chat_messages(world: &mut E2eWorld) -> Vec { .is_some_and(|messages| { messages.iter().any(|m| { m.get("role").and_then(serde_json::Value::as_str) == Some("user") - && message_text(m.get("content").unwrap_or(&serde_json::Value::Null)) - .contains(MANAGED_MODEL_PROMPT) + && chat_message_text( + m.get("content").unwrap_or(&serde_json::Value::Null), + ) + .contains(MANAGED_MODEL_PROMPT) }) }) }) @@ -2464,32 +2561,10 @@ async fn recorded_chat_messages(world: &mut E2eWorld) -> Vec { .unwrap_or_else(|| panic!("chat request had no messages array:\n{body}")) .iter() .filter_map(|m| m.get("content")) - .map(message_text) + .map(chat_message_text) .collect() } -/// The text of one OpenAI-format message. `content` is a bare string on the -/// turns the TUI builds, but an array of typed parts on the system message the -/// chat client emits — read both, or the grounding looks absent when it is -/// simply wrapped. -fn message_text(content: &serde_json::Value) -> String { - content.as_str().map_or_else( - || { - content - .as_array() - .map(|parts| { - parts - .iter() - .filter_map(|p| p.get("text").and_then(serde_json::Value::as_str)) - .collect::>() - .join("\n") - }) - .unwrap_or_default() - }, - str::to_owned, - ) -} - /// The single line of the sent prompt that opens with `label`, or a failure /// naming what was actually sent. Asserting on the request — never on the /// canned reply — is the point: the mock answers identically whatever it is diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 6e8d5aebfa..c7fac668d6 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -48,13 +48,13 @@ const PREVIEW_FIX_ID: &str = "fix-1-arch"; /// found". This one needs only `--device-index`, which the scenario supplies. const MUTATING_FIX_ID: &str = "fix-9-igpu-dgpu"; -/// The recipe used to prove a failed helper command is explained on stderr with -/// exit code 4. `fix-4-render-group` is the only AUTO recipe whose command-failure -/// branch this suite can force deterministically: its helper (`usermod`, run -/// directly as root or via `sudo` otherwise) is resolved off `$PATH`, so a -/// scenario-controlled `$PATH` (see `command_fails_bin_dir`) can stand a fake -/// `usermod`/`sudo` in for it, unconditionally failing, without needing real -/// root or touching real group membership. +/// The recipe used to prove a failed or timed-out helper command is explained +/// on stderr with exit code 4. `fix-4-render-group` is the only AUTO recipe whose +/// command-failure branch this suite can force deterministically: its helper +/// (`usermod`, run directly as root or via `sudo` otherwise) is resolved off +/// `$PATH`, so a scenario-controlled `$PATH` (see `command_fails_bin_dir`) can +/// stand a fake `usermod`/`sudo` in for it, failing or never finishing, without +/// needing real root or touching real group membership. const COMMAND_FAILURE_FIX_ID: &str = "fix-4-render-group"; /// The Windows PATH-repair recipe at the center of ROCMAI-449. Its own e2e @@ -274,6 +274,26 @@ fn command_fails_bin_dir(world: &E2eWorld) -> std::path::PathBuf { .join("fake-bin") } +/// Where the fake `usermod`/`sudo` record each call, one ` ` line +/// per call. Outside [`command_fails_bin_dir`] so it is not on the CLI's PATH. +fn render_group_helper_log(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("fake-bin-calls.log") +} + +/// The `$USER` the render-group scenarios run the fix as. +const RENDER_GROUP_USER: &str = "e2e-test-user"; + +/// The line `fix-4-render-group` prints once the user was added. Built from +/// the same user the scenario sets, and absent from every failure path. +fn render_group_success_line() -> String { + format!("Added {RENDER_GROUP_USER} to render,video.") +} + // ── Given ────────────────────────────────────────────────────────── #[given("a user who hit a known ROCm failure")] @@ -572,23 +592,47 @@ async fn user_chose_fix_for_another_os(world: &mut E2eWorld) { #[given("a user who has approved a fix whose helper command will fail")] async fn user_approved_fix_that_will_fail(world: &mut E2eWorld) { + plant_render_group_helpers(world, "exit 1"); +} + +#[given("a user who has approved a fix whose helper command never finishes")] +async fn user_approved_fix_that_never_finishes(world: &mut E2eWorld) { + // `exec`, so the process `rocm` started *is* the sleep: the executor stops + // only its direct child, and a `sleep` left as the shell's own child would + // outlive the scenario by ten minutes. The absolute path because the + // scenario's PATH holds only the stand-ins. + plant_render_group_helpers(world, "exec /bin/sleep 600"); +} + +/// Stands fake `usermod`/`sudo` scripts in for the fix's helper on a `$PATH` +/// holding nothing else. Each appends ` ` to +/// [`render_group_helper_log`] and then runs `tail`, so a scenario can say +/// both what the CLI reported and what it asked the helper to do. +fn plant_render_group_helpers(world: &mut E2eWorld, tail: &str) { let bin_dir = command_fails_bin_dir(world); std::fs::create_dir_all(&bin_dir).expect("failed to create the scenario's fake PATH dir"); + let log = render_group_helper_log(world); + let log = log.to_str().expect("the scenario root is UTF-8"); + assert!( + !log.contains('\''), + "the helper log path is single-quoted in the stand-in script: {log}" + ); // `usermod` only has to exist for the `which` probe; the command actually run // -- `usermod` directly if already root, `sudo usermod ...` otherwise -- goes - // through one of these two scripts either way, and both fail unconditionally. + // through one of these two scripts either way. for name in ["usermod", "sudo"] { let script = bin_dir.join(name); - rocm_core::fake_executable::write_fake_executable(&script, "#!/bin/sh\nexit 1\n") + let body = format!("#!/bin/sh\necho \"{name} $*\" >> '{log}'\n{tail}\n"); + rocm_core::fake_executable::write_fake_executable(&script, body) .unwrap_or_else(|e| panic!("failed to write fake {name}: {e}")); } // Restricting `$PATH` to only the fakes above also makes the recipe's // root-detection deterministic: it shells out to `id -u`, which is not on // this PATH, so the spawn itself fails and reads as "not root" regardless of - // who runs the suite -- the same `sudo usermod` branch fails on every host, + // who runs the suite -- the same `sudo usermod` branch runs on every host, // CI or developer machine, root or not. world.command_env.push(("PATH", bin_dir.into_os_string())); - world.command_env.push(("USER", "e2e-test-user".into())); + world.command_env.push(("USER", RENDER_GROUP_USER.into())); world.model_name = Some(COMMAND_FAILURE_FIX_ID.to_string()); } @@ -628,16 +672,41 @@ async fn user_on_windows_with_hip_sdk_and_no_powershell(world: &mut E2eWorld) { world .command_env .push(("HIP_PATH", path_fix_sdk_dir(world).into_os_string())); + deny_powershell(world); + world.model_name = Some(PATH_FIX_ID.to_string()); +} + +#[given("a Windows host where PowerShell cannot be found")] +async fn windows_host_without_powershell(world: &mut E2eWorld) { + deny_powershell(world); +} + +#[given("a user who has chosen the fix that hides the integrated GPU")] +async fn user_chose_igpu_fix(world: &mut E2eWorld) { + world.model_name = Some(MUTATING_FIX_ID.to_string()); +} + +/// Gives the CLI an empty `PATH`, so its own PowerShell reads cannot start. +/// +/// It denies PowerShell only. The CLI starts both programs by bare name, and +/// on Windows Rust's `Command` looks for a bare name in the CLI's own +/// directory, then the system directory (System32), then the Windows +/// directory, and only then in the CLI's `PATH` (empty here). `setx.exe` is in +/// System32, so it is still found; `powershell.exe` is in a subdirectory of +/// System32 that only `PATH` reaches, so it is not. What keeps each +/// scenario off the runner's real User environment, should the read guard +/// regress, is the scenario's own: fix-6 (diagnose-39) writes through +/// PowerShell, which cannot start; fix-2 (diagnose-69) would read both scopes +/// as unset and find nothing to clear; fix-9 (diagnose-70) is only previewed. +fn deny_powershell(world: &mut E2eWorld) { // An empty child PATH means the CLI's own `Command::new("powershell")` // cannot resolve the binary at all -- `powershell.exe` lives under // `System32\WindowsPowerShell\v1.0`, which Windows does NOT search // unconditionally the way it does System32 itself, so this is a real // instance of the "PowerShell missing" cause `ps_env_scope`'s doc comment - // names, not a faked one. Nothing here touches the real machine's - // User-scope PATH, so no capture/restore is needed: the read never gets - // far enough to risk it. + // names, not a faked one. No capture/restore is needed: see above for why + // none of the three scenarios can write the real User environment. world.command_env.push(("PATH", std::ffi::OsString::new())); - world.model_name = Some(PATH_FIX_ID.to_string()); } // ── When ─────────────────────────────────────────────────────────── @@ -728,7 +797,25 @@ async fn user_applies_fix_without_agreeing(world: &mut E2eWorld) { #[when("the user asks the CLI to apply the approved fix")] async fn user_applies_approved_fix(world: &mut E2eWorld) { let fix_id = world.model_name.clone().expect("no fix id set"); - let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &["fix", &fix_id, "--yes"]); + // Off the executor: in diagnose-68 the fix waits out its one-minute + // timeout, which would otherwise stall every concurrent scenario. + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env_off_executor(world, &["fix", &fix_id, "--yes"]).await; + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user previews that fix for the second GPU")] +async fn user_previews_fix_for_second_gpu(world: &mut E2eWorld) { + let fix_id = world.model_name.clone().expect("no fix id set"); + // `--dry-run` and no `--yes`: even a fix that wrongly went past its read + // would stop at the preview, and failing that, at the consent gate (stdin + // is piped), before running `setx` on the runner. + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["fix", &fix_id, "--device-index", "1", "--dry-run"], + ); world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); world.cli_rc = Some(rc); @@ -1728,7 +1815,7 @@ async fn assert_command_failure_reported_on_stderr(world: &mut E2eWorld) { ); let stderr = world.cli_stderr.as_deref().unwrap_or(""); assert!( - stderr.contains("usermod exited") && stderr.contains("group membership NOT changed"), + stderr.contains("usermod exited 1; group membership NOT changed."), "the command-failure explanation must be on stderr:\n{stderr}" ); let stdout = world.cli_output.as_deref().unwrap_or(""); @@ -1736,6 +1823,83 @@ async fn assert_command_failure_reported_on_stderr(world: &mut E2eWorld) { !stdout.contains("group membership NOT changed"), "the command-failure explanation must not also be on stdout:\n{stdout}" ); + // "NOT changed" is a claim about state: it holds only for a change that + // was asked for, refused, and not then reported as made. + assert!( + !stdout.contains(&render_group_success_line()), + "a failed helper must not be followed by the success line:\n{stdout}" + ); + assert_render_group_helper_was_asked(world); +} + +#[then( + "the CLI reports on stderr, with exit code 4, that the helper was stopped after a minute, how to check the change and which fix to re-run" +)] +async fn assert_command_timeout_reported_on_stderr(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + assert_eq!( + world.cli_rc, + Some(4), + "a helper stopped at the deadline is a failed attempt, exit 4:\n{stdout}\n{stderr}" + ); + // Not run as root (see `plant_render_group_helpers`), so the command is + // `sudo usermod`; the bound is the fix's own one-minute RUN_TIMEOUT. + let expected = format!( + "sudo usermod timed out after 60s and was stopped; group membership may or may not have \ + changed. Check with `id -nG {RENDER_GROUP_USER}`, then re-run \ + `rocm fix {COMMAND_FAILURE_FIX_ID}` if it did not." + ); + assert!( + stderr.lines().any(|line| line == expected), + "expected the timeout line on stderr:\n {expected}\ngot:\n{stderr}" + ); + assert!( + !stdout.contains("timed out"), + "the timeout explanation must not also be on stdout:\n{stdout}" + ); +} + +#[then("the CLI claims neither that the change was made nor that nothing changed")] +async fn assert_no_claim_about_the_unfinished_change(world: &mut E2eWorld) { + let combined = format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ); + // The CLI stopped `sudo`, not `usermod`, so either claim could be false. + assert!( + !combined.contains("NOT changed"), + "a stopped helper may have made the change; the CLI must not say it did not:\n{combined}" + ); + assert!( + !combined.contains(&render_group_success_line()), + "a stopped helper may not have made the change; the CLI must not say it did:\n{combined}" + ); +} + +#[then("the helper was asked to add the user to the render and video groups")] +async fn assert_render_group_helper_asked(world: &mut E2eWorld) { + assert_render_group_helper_was_asked(world); +} + +/// The stand-in's log shows the CLI asked to add [`RENDER_GROUP_USER`] to the +/// render and video groups, via `sudo` or directly. +fn assert_render_group_helper_was_asked(world: &E2eWorld) { + let log_path = render_group_helper_log(world); + let log = std::fs::read_to_string(&log_path).unwrap_or_else(|e| { + panic!( + "the helper stand-in was never run ({}: {e}), so the reported outcome is about a \ + command that did not happen", + log_path.display() + ) + }); + let asked = format!("usermod -a -G render,video {RENDER_GROUP_USER}"); + assert!( + log.lines() + .any(|line| line == format!("sudo {asked}") || line == asked), + "expected the helper to be asked `{asked}`; it recorded:\n{log}" + ); } #[then("the CLI used PowerShell's SetEnvironmentVariable, not setx, to update the User PATH")] @@ -1791,24 +1955,104 @@ async fn assert_path_fix_prepended_with_apostrophe_intact(world: &mut E2eWorld) #[then("the CLI aborts the PATH fix with exit code 4 and changes nothing")] async fn assert_path_fix_aborts_on_unreadable_path(world: &mut E2eWorld) { - let output = world.cli_output.as_deref().unwrap_or(""); + // Exit 4 -- the same code as a write failure -- rather than a failed read + // being silently treated as an empty PATH (ROCMAI-449). + assert_read_failure_stops_the_fix( + world, + "PATH", + &["Plan: prepend", "This only takes effect in NEW shells"], + ); +} + +#[then( + "the CLI stops with exit code 4, saying it could not read the saved override and that nothing was changed" +)] +async fn assert_override_fix_aborts_on_unreadable_scope(world: &mut E2eWorld) { + assert_read_failure_stops_the_fix( + world, + "HSA_OVERRIDE_GFX_VERSION", + &["Cleared from User scope"], + ); +} + +#[then("the CLI does not report the override as already gone")] +async fn assert_override_not_reported_gone(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + for claim in ["No persistent HSA_OVERRIDE_GFX_VERSION", "You're done"] { + assert!( + !stdout.contains(claim), + "a value that could not be read must not be reported as absent ({claim:?}):\n{stdout}" + ); + } +} + +#[then( + "the CLI stops with exit code 4, saying it could not read the saved device setting and that nothing was changed" +)] +async fn assert_igpu_fix_aborts_on_unreadable_scope(world: &mut E2eWorld) { + // Nothing after the read is listed here: this is a dry run, so the lines a + // fix that went on would print are the preview, which the next step checks. + assert_read_failure_stops_the_fix(world, "HIP_VISIBLE_DEVICES", &[]); +} + +#[then("the fix's catalog entry is printed, but not the device setting it would save")] +async fn assert_no_device_setting_plan(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + // `rocm fix` prints the catalog entry before the runner reads anything, + // so a stopped dry run still shows it; the docs say exactly that. The + // `Fix:` line is the one `print_recipe` opens the entry with. + let fix_id = world.model_name.as_deref().expect("no fix id set"); + assert!( + stdout + .lines() + .any(|line| line.starts_with("Fix:") && line.contains(fix_id)), + "the catalog entry for {fix_id} must still be printed on stdout:\n{stdout}" + ); + // The runner's own preview, whatever its heading says: the concrete + // `setx` for the index the When step passed and the dry-run marker. The + // catalog entry's `Commands:` lists `setx HIP_VISIBLE_DEVICES `, + // a template, so only the concrete index is forbidden. + for line in [ + "Plan: persist HIP_VISIBLE_DEVICES", + "already sets HIP_VISIBLE_DEVICES", + "setx HIP_VISIBLE_DEVICES 1", + "(dry-run; not executed)", + ] { + assert!( + !stdout.contains(line), + "a failed read must stop the fix before it plans or judges a write ({line:?}):\n{stdout}" + ); + } +} + +/// A Windows fix whose User-scope read of `var` could not complete exits 4 +/// with the one line that says so and that nothing was changed, and prints +/// none of `after`, the lines only a fix that went on would print. +fn assert_read_failure_stops_the_fix(world: &E2eWorld, var: &str, after: &[&str]) { + let stdout = world.cli_output.as_deref().unwrap_or(""); let stderr = world.cli_stderr.as_deref().unwrap_or(""); assert_eq!( world.cli_rc, Some(4), - "a failed PATH read must abort with exit code 4 -- the same code as a \ - write failure -- rather than being silently treated as an empty PATH \ - (ROCMAI-449):\n{output}\n{stderr}" + "a read that did not complete must stop the fix with exit 4:\n{stdout}\n{stderr}" ); - let combined = format!("{output}\n{stderr}"); + // How PowerShell failed to start is the host's to say (not found, or a + // spawn error), so the middle of the line is not pinned. + let start = format!("Could not read the current User {var}: PowerShell "); + let end = "; nothing was changed."; assert!( - combined.contains("Could not read the current User PATH"), - "expected the abort message naming the failed read:\n{combined}" - ); - assert!( - !combined.contains("Plan: prepend"), - "a failed read must abort before any write plan is printed:\n{combined}" + stderr + .lines() + .any(|line| line.starts_with(&start) && line.ends_with(end)), + "expected a stderr line `{start}...{end}`:\n{stderr}" ); + let combined = format!("{stdout}\n{stderr}"); + for line in after { + assert!( + !combined.contains(line), + "a failed read must stop the fix before {line:?}:\n{combined}" + ); + } } // ── `rocm diagnose --model` ──────────────────────────────────────── diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a986bc22f7..a3b34fef3c 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -546,6 +546,54 @@ fn planted_setup_runtime_root(world: &E2eWorld) -> std::path::PathBuf { .join("setup-runtime") } +#[given("the active runtime is a nightly runtime installed for one exact GPU")] +async fn active_runtime_is_nightly_exact_target(world: &mut E2eWorld) { + use super::runtime_lifecycle_steps::{ + NIGHTLY_EXACT_TARGET_KEY, plant_nightly_exact_target_runtime, + }; + plant_nightly_exact_target_runtime(world); + // Black-box: plain JSON matching the CLI's on-disk config schema; every + // other field defaults. `examine` reads the active runtime from here. + let config_dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("config"); + std::fs::create_dir_all(&config_dir).expect("failed to create config dir"); + let config = serde_json::json!({ "active_runtime_key": NIGHTLY_EXACT_TARGET_KEY }); + std::fs::write( + config_dir.join("config.json"), + serde_json::to_string_pretty(&config).expect("failed to serialize config"), + ) + .expect("failed to write config"); +} + +#[then( + "the runtime state names the channel, package source, source layout and exact GPU target that runtime was installed from" +)] +async fn runtime_state_names_active_provenance(world: &mut E2eWorld) { + use super::runtime_lifecycle_steps::{ + NIGHTLY_DEVICE_TARGET, NIGHTLY_SOURCE, NIGHTLY_SOURCE_LAYOUT, + }; + let out = world.cli_output.clone().unwrap_or_default(); + for (field, expected) in [ + ("active_runtime_channel", "nightly"), + ("active_runtime_source", NIGHTLY_SOURCE), + ( + "active_runtime_source_layout_generation", + NIGHTLY_SOURCE_LAYOUT, + ), + ("active_runtime_device_target", NIGHTLY_DEVICE_TARGET), + ] { + assert_eq!( + field_value(&out, field), + Some(expected), + "examine must report {field} for the active runtime:\n{out}" + ); + } +} + #[given("setup names a runtime folder the registry has forgotten")] async fn setup_names_folder_registry_forgot(world: &mut E2eWorld) { let root = world.isolated_root.as_ref().expect("no isolated root"); @@ -3182,3 +3230,209 @@ async fn assert_inspection_agrees_with_the_refusal(world: &mut E2eWorld) { "{json:#}" ); } + +/// What the planted runtime's own `librocdxg.so.1` holds, so a read through a link +/// shows which file the link points at. +const PLANTED_RUNTIME_ROCDXG: &str = "the runtime's own librocdxg"; + +/// A planted managed runtime makes every `rocm` start run the startup update +/// check, which needs a PATH Python and the network and prints progress ahead of a +/// `--json` document. The scenarios that plant one are about the host, not +/// updates. Like every scenario env it is consumed by one run, so a step that runs +/// `rocm` again re-plants it. +fn disable_startup_update_check(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1".into())); +} + +/// Plant a managed wheel runtime in the registry, with the library directory its +/// HSA runtime lives in. `with_unversioned` also plants `librocdxg.so` beside the +/// versioned copy, which is the layout where the runtime loads its own copy. +/// +/// Black-box: plain JSON in the registry schema, as `rocm install sdk` would +/// have written it, with files standing in for the libraries (an empty file is +/// enough for a check of which names exist). +fn plant_managed_runtime_with_rocdxg(world: &mut E2eWorld, with_unversioned: bool) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let lib = root.join("wheel-runtime/lib"); + std::fs::create_dir_all(&lib).expect("failed to create the planted runtime lib"); + std::fs::write(lib.join("libhsa-runtime64.so.1"), "").expect("failed to plant the runtime"); + std::fs::write(lib.join("librocdxg.so.1"), PLANTED_RUNTIME_ROCDXG) + .expect("failed to plant librocdxg.so.1"); + if with_unversioned { + std::fs::write(lib.join("librocdxg.so"), "").expect("failed to plant librocdxg.so"); + } + let registry = root.join("data/runtimes/registry"); + std::fs::create_dir_all(®istry).expect("failed to create the runtime registry"); + let manifest = serde_json::json!({ + "runtime_key": "wsl-rocdxg-runtime", + "runtime_id": "therock-nightly:gfx1151", + "channel": "nightly", + "format": "wheel", + "family": "gfx1151", + "family_source": "e2e", + "version": "10.3.0a20261113", + "install_root": root.join("wheel-runtime").display().to_string(), + "selected_artifact_url": "https://example.invalid/e2e.whl", + "installed_at_unix_ms": 1, + "rocm_sdk": { + "import_ok": true, + "root_path": root.join("wheel-runtime").display().to_string(), + "bin_path": root.join("wheel-runtime/bin").display().to_string(), + "library_paths": [lib.display().to_string()], + }, + }); + std::fs::write( + registry.join("wsl-rocdxg-runtime.json"), + serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), + ) + .expect("failed to write the planted runtime manifest"); + disable_startup_update_check(world); +} + +#[given("a WSL machine with a managed runtime that ships only a versioned librocdxg")] +async fn wsl_machine_with_runtime_shipping_only_versioned_rocdxg(world: &mut E2eWorld) { + plant_wsl_machine_with_unregistered_rocdxg(world, "/opt/rocm/lib"); + plant_managed_runtime_with_rocdxg(world, false); +} + +#[given("a machine that is not WSL with a managed runtime that ships only a versioned librocdxg")] +async fn native_machine_with_runtime_shipping_only_versioned_rocdxg(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + // A simulated host root with no `/dev/dxg` and no Microsoft kernel string. + std::fs::create_dir_all(root.join("host-root/proc")).expect("failed to create planted /proc"); + plant_managed_runtime_with_rocdxg(world, false); + world.command_env.push(( + "ROCM_CLI_TEST_HOST_ROOT", + root.join("host-root").into_os_string(), + )); +} + +#[given("a WSL machine with a managed runtime whose librocdxg.so is a link to a missing file")] +async fn wsl_machine_with_runtime_holding_a_dangling_rocdxg_link(world: &mut E2eWorld) { + plant_wsl_machine_with_unregistered_rocdxg(world, "/opt/rocm/lib"); + plant_managed_runtime_with_rocdxg(world, false); + let lib = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("wheel-runtime/lib"); + // The loader's open() fails on a broken link as on a missing file, so this is + // as good as no link: the runtime loads the host's librocdxg. + plant_dangling_link("gone.so.9", &lib.join("librocdxg.so")); +} + +/// A link to a file that does not exist. The scenario is `@requires-os:linux`, so +/// the non-Unix arm exists only to keep the harness compiling on Windows. +#[cfg(unix)] +fn plant_dangling_link(target: &str, link: &std::path::Path) { + std::os::unix::fs::symlink(target, link).expect("failed to plant a dangling link"); +} + +#[cfg(not(unix))] +fn plant_dangling_link(_target: &str, _link: &std::path::Path) { + unreachable!("a Linux-only scenario step"); +} + +#[given("a WSL machine with a managed runtime that has librocdxg.so beside its own copy")] +async fn wsl_machine_with_runtime_holding_unversioned_rocdxg(world: &mut E2eWorld) { + plant_wsl_machine_with_unregistered_rocdxg(world, "/opt/rocm/lib"); + plant_managed_runtime_with_rocdxg(world, true); +} + +fn planted_runtime_librocdxg_copy(world: &E2eWorld) -> String { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("wheel-runtime/lib/librocdxg.so.1") + .display() + .to_string() +} + +fn unlinked_runtime_librocdxg(world: &E2eWorld) -> Vec { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + json.pointer("/summary/unlinked_runtime_librocdxg") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("no unlinked_runtime_librocdxg list:\n{json:#}")) + .iter() + .filter_map(|path| path.as_str().map(str::to_owned)) + .collect() +} + +#[then("the inspection lists that runtime's librocdxg as one its own runtime does not load")] +async fn assert_runtime_librocdxg_listed(world: &mut E2eWorld) { + assert_eq!( + unlinked_runtime_librocdxg(world), + vec![planted_runtime_librocdxg_copy(world)] + ); +} + +#[then("the inspection lists no runtime librocdxg that its own runtime does not load")] +async fn assert_runtime_librocdxg_not_listed(world: &mut E2eWorld) { + assert_eq!(unlinked_runtime_librocdxg(world), Vec::::new()); +} + +#[when( + "the user runs the command the inspection advises for that runtime's librocdxg and inspects again" +)] +async fn user_follows_the_runtime_librocdxg_advice(world: &mut E2eWorld) { + // The scenario env is consumed by each run; a scenario that inspected before + // this step needs the machine planted again. + if world.command_env.is_empty() { + push_planted_wsl_machine_env(world, "/opt/rocm/lib"); + disable_startup_update_check(world); + } + let (text, _, text_rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); + assert_eq!(text_rc, 0, "`rocm examine` failed:\n{text}"); + let line = text + .lines() + .find(|line| line.trim_start().starts_with("wsl_runtime_librocdxg:")) + .unwrap_or_else(|| panic!("the inspection has no wsl_runtime_librocdxg line:\n{text}")); + let command = line.rsplit_once("run: ").map_or_else( + || panic!("the line names no command: {line}"), + |(_, command)| command.trim(), + ); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(command) + .status() + .expect("failed to run the advised command"); + assert!(status.success(), "the advised command failed: {command}"); + // Clearing the finding is not enough: a link to the wrong file would clear it + // too. The unversioned name must read as the runtime's own copy. + let link = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("wheel-runtime/lib/librocdxg.so"); + assert_eq!( + std::fs::read_to_string(&link).expect("librocdxg.so does not resolve"), + PLANTED_RUNTIME_ROCDXG + ); + // The scenario env is consumed by each run (`mem::take` in + // `run_rocm_with_scenario_env`), so the second run needs it planted again. + push_planted_wsl_machine_env(world, "/opt/rocm/lib"); + disable_startup_update_check(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip", "--json"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 10a98f0295..3ddc414af7 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -8,7 +8,8 @@ //! Black-box throughout. The two Lemonade trees and the install record are //! planted as plain files in the layout the engine writes, so no scenario //! downloads anything; the refusal scenario reaches the engine's free-space -//! check through the `rocm/e2e-test-hooks` override instead of a full drive. +//! check through the `rocm/e2e-test-hooks` override instead of a full drive, +//! and the host-library refusal through the same build's library override. use std::path::PathBuf; @@ -143,6 +144,196 @@ async fn assert_lemonade_install_refused(world: &mut E2eWorld) { ); } +/// An Ubuntu 20.04-like host, as the probe reads one: glibc 2.31, and no +/// `libssl.so.3` in the loader cache because it has only OpenSSL 1.1. Reported +/// through the `rocm/e2e-test-hooks` override, so the refusal is reached on a +/// host whose real glibc and OpenSSL are new. +const OLD_HOST_GLIBC: &str = "2.31"; +const OLD_HOST_OPENSSL: &str = "missing"; + +#[given("this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3")] +async fn host_libraries_too_old(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_E2E_LEMONADE_HOST_GLIBC", OLD_HOST_GLIBC.into())); + world + .command_env + .push(("ROCM_E2E_LEMONADE_HOST_OPENSSL", OLD_HOST_OPENSSL.into())); +} + +/// The refusal names, for each library, the version needed and what the host +/// has (a version, or no `libssl.so.3`), says nothing was downloaded, and that +/// is true on disk. +#[then("the Lemonade install is refused for the host's libraries before anything is downloaded")] +async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld) { + let output = combined_output(world); + // Without the hook the engine sees this host's real libraries and starts a + // real install. Name that cause rather than a baffling mismatch. + assert!( + output.contains("this host's libraries are too old for the Lemonade "), + "no host-library refusal: the binary under test was probably built without \ + the `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail on a host too old for the pinned Lemonade server:\n{output}" + ); + for expected in [ + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), + "or newer (this host's loader cache has no libssl.so.3)".to_owned(), + "it needs glibc ".to_owned(), + "Nothing was downloaded.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + let tree = data_folder_tree(world); + assert!( + !tree.join("downloads").exists(), + "an archive was downloaded despite the refusal:\n{output}" + ); + assert!( + !tree.join("runtime").exists(), + "a runtime was extracted despite the refusal:\n{output}" + ); + assert!( + !install_record(world).exists(), + "a refused install must not record Lemonade as installed:\n{output}" + ); +} + +#[given("the user has turned off the Lemonade host-library check")] +async fn host_library_check_turned_off(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK", "1".into())); +} + +/// The warning names the shortfall and the variable that let the install past +/// it; the next Then shows the install really went on (to the space check). +#[then("the user is warned that the host's libraries are too old and the install went ahead")] +async fn assert_host_library_warning(world: &mut E2eWorld) { + let output = combined_output(world); + assert!( + output.contains("warning: this host's libraries are too old for the Lemonade "), + "no host-library warning: the binary under test was probably built without \ + the `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + for expected in [ + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), + "going ahead because ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + assert!( + !output.contains("to skip this check."), + "the install was refused for the host's libraries despite the opt-out:\n{output}" + ); +} + +/// The Lemonade version this build pins, read black-box from the refusal the +/// old-host hooks produce on an empty data folder (lemonade-installs-07 shows +/// that run changes nothing): "... too old for the Lemonade server". +/// +/// A run consumes the scenario's environment, so the old-host hooks the Given +/// before this one set are put back for the When. +fn pinned_lemonade_version(world: &mut E2eWorld) -> String { + let scenario_env = world.command_env.clone(); + let (stdout, stderr, _) = + crate::run_rocm_with_scenario_env(world, &["engines", "install", "lemonade", "--yes"]); + world.command_env = scenario_env; + let output = format!("{stdout}\n{stderr}"); + output + .split("too old for the Lemonade ") + .nth(1) + .and_then(|rest| rest.split_whitespace().next()) + .unwrap_or_else(|| { + panic!( + "no host-library refusal to read the pinned version from; the binary under \ + test was probably built without the `rocm/e2e-test-hooks` feature:\n{output}" + ) + }) + .to_owned() +} + +/// What the planted tree holds, file by file, so the Then can show it was left +/// exactly as it was. +fn tree_files(root: &std::path::Path) -> Vec<(PathBuf, Vec)> { + let mut files = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).expect("failed to list the planted tree") { + let path = entry.expect("failed to read a tree entry").path(); + if path.is_dir() { + stack.push(path); + } else { + let bytes = std::fs::read(&path).expect("failed to read a tree file"); + files.push((path, bytes)); + } + } + } + files.sort(); + files +} + +#[given("the pinned Lemonade is already installed in the data folder")] +async fn plant_pinned_lemonade_install(world: &mut E2eWorld) { + let version = pinned_lemonade_version(world); + let tree = data_folder_tree(world); + plant_lemonade_tree(&tree, DATA_FOLDER_TREE_BYTES); + // The marker `prepare_embeddable` writes after a complete extraction. + std::fs::write(tree.join("runtime").join("runtime-version.txt"), &version) + .expect("failed to write the runtime version marker"); + world.planted_tree_snapshot = Some(tree_files(&tree)); +} + +#[then( + "the Lemonade install is refused for the host's libraries and the installed copy is left as it is" +)] +async fn assert_refused_and_installed_copy_kept(world: &mut E2eWorld) { + let output = combined_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail on a host too old for the pinned Lemonade server:\n{output}" + ); + let tree = data_folder_tree(world); + let runtime_dir = tree.join("runtime"); + let expected = format!( + "already installed in {} stays as it is", + runtime_dir.display() + ); + for expected in [ + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), + "Nothing was changed: the Lemonade ".to_owned(), + expected, + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + assert!( + !output.contains("Nothing was downloaded."), + "the refusal describes a download the re-run would never have made:\n{output}" + ); + // "Nothing was changed" is checked against the disk. + let before = world + .planted_tree_snapshot + .as_ref() + .expect("no planted tree snapshot - did the Given run?"); + assert!( + &tree_files(&tree) == before, + "the installed tree changed despite the refusal:\n{output}" + ); +} + #[then("the user is told what the Lemonade install needs and where it would go")] async fn assert_lemonade_install_notice(world: &mut E2eWorld) { let output = combined_output(world); diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index aa3d3d0fd4..929f91af55 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -17,6 +17,7 @@ //! by [`LifecycleState`]'s `Drop`, even when a step fails. use std::fmt::Write as _; +use std::num::NonZeroU16; use std::path::{Path, PathBuf}; use std::process::Command; @@ -60,12 +61,42 @@ pub struct LifecycleState { /// A shell profile the installer is pointed at through /// `ROCM_CLI_SHELL_PROFILE`; `None` means the default `~/.bashrc`. custom_profile: Option, + /// Installer output captured straight off a pseudo-terminal, carriage + /// returns and all. `last_output` cannot serve here: it comes from a piped + /// `Command`, which the installer sees as a non-terminal and answers with + /// plain lines, so the progress drawing this holds never appears in it. + last_terminal_output: String, + /// The same bytes after a terminal emulator has drawn them, one row per + /// line. Wrapping is something the emulator does when a line overruns the + /// margin, never something the writer puts in the byte stream, so a torn + /// progress bar is only visible once the stream has been rendered. Raw + /// bytes answer "how wide was each frame", this answers "what did the user + /// end up looking at". + last_terminal_screen: String, + /// The pty width the last run used. Assertions read this rather than the + /// `PTY_COLS` constant, so a scenario driving a different width is checked + /// against the terminal it actually had instead of silently against 80. + last_pty_cols: u16, /// The login shell (`$SHELL`) the installer is run under; `None` leaves the /// runner's own. login_shell: Option, /// The fish config seeded before install, which uninstall must restore /// exactly. fish_config_before_install: Option, + /// What the installer is told about `ROCM_CLI_REQUIRE_SIGNATURE`. + signature_requirement: SignatureRequirement, +} + +/// What a scenario's installer runs are told about `ROCM_CLI_REQUIRE_SIGNATURE`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum SignatureRequirement { + /// Set to `1`, as every scenario but the default-trust ones runs. + #[default] + Required, + /// Not set at all, as a user who never asked for a signature runs. + Unset, + /// Set to `0`, as a user who turned the requirement off runs. + Off, } /// The Windows user `Path` exactly as the registry holds it: its value with @@ -210,9 +241,29 @@ fn run_installer( download_base: &str, public_key_env: &[(&str, String)], extra_env: &[(&str, String)], +) -> (String, bool) { + run_installer_on_channel( + world, + "release", + install_dir, + download_base, + public_key_env, + extra_env, + ) +} + +/// [`run_installer`] with the channel argument the installer is given: `release` +/// or a release tag naming a specific version. +fn run_installer_on_channel( + world: &E2eWorld, + channel: &str, + install_dir: &Path, + download_base: &str, + public_key_env: &[(&str, String)], + extra_env: &[(&str, String)], ) -> (String, bool) { let root = workspace_root(); - let cmd = installer_command(&root); + let cmd = installer_command(&root, channel); run_installer_command( world, cmd, @@ -237,13 +288,30 @@ fn run_installer_command( cmd.current_dir(current_dir); cmd.env("ROCM_CLI_DOWNLOAD_BASE", download_base); cmd.env("ROCM_CLI_INSTALL_DIR", install_dir); - cmd.env("ROCM_CLI_REQUIRE_SIGNATURE", "1"); + match st.signature_requirement { + SignatureRequirement::Required => { + cmd.env("ROCM_CLI_REQUIRE_SIGNATURE", "1"); + } + // Also drops one inherited from the runner, so the install sees the + // variable genuinely unset. + SignatureRequirement::Unset => { + cmd.env_remove("ROCM_CLI_REQUIRE_SIGNATURE"); + } + SignatureRequirement::Off => { + cmd.env("ROCM_CLI_REQUIRE_SIGNATURE", "0"); + } + } // Isolate the installer's HOME so shell-profile writes and the seeded // ~/.rocm/config.json land in the scenario's tree, never the real home. cmd.env("HOME", &st.home_dir); // fish's config lives under $XDG_CONFIG_HOME when that is set, which would // point the installer's fish setup at the runner's real config. cmd.env_remove("XDG_CONFIG_HOME"); + // The installer takes a key from these in preference to its pinned ones, so + // one inherited from the runner would stand in for the pinned key on every + // path that supplies none. The child sees only the key the step passes. + cmd.env_remove("ROCM_CLI_SIGNING_PUBLIC_KEY_PATH"); + cmd.env_remove("ROCM_CLI_SIGNING_PUBLIC_KEY_PEM"); for (key, value) in public_key_env { cmd.env(key, value); } @@ -259,22 +327,28 @@ fn run_installer_command( (combined, output.status.success()) } -/// Build the platform-appropriate installer command (`sh install.sh release` on -/// Unix; `pwsh -File install.ps1 release` on Windows). +/// Build the platform-appropriate installer command for `channel` +/// (`sh install.sh ` on Unix; `pwsh -File install.ps1 ` on +/// Windows). #[cfg(not(windows))] -fn installer_command(root: &Path) -> Command { +fn installer_command(root: &Path, channel: &str) -> Command { let mut cmd = Command::new("sh"); - cmd.arg(root.join("install.sh")).arg("release"); + cmd.arg(root.join("install.sh")).arg(channel); cmd } #[cfg(windows)] -fn installer_command(root: &Path) -> Command { - powershell_installer_command(&root.join("install.ps1")) +fn installer_command(root: &Path, channel: &str) -> Command { + powershell_command(&root.join("install.ps1"), channel) } #[cfg(windows)] fn powershell_installer_command(installer: &Path) -> Command { + powershell_command(installer, "release") +} + +#[cfg(windows)] +fn powershell_command(installer: &Path, channel: &str) -> Command { // Prefer pwsh; fall back to Windows PowerShell. let shell = if which_ok("pwsh") { "pwsh" @@ -284,7 +358,7 @@ fn powershell_installer_command(installer: &Path) -> Command { let mut cmd = Command::new(shell); cmd.args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-File"]) .arg(installer) - .arg("release"); + .arg(channel); cmd } @@ -543,8 +617,12 @@ async fn given_release_tree(world: &mut E2eWorld) { smoke_cache: None, profile_before_install: None, custom_profile: None, + last_terminal_output: String::new(), + last_terminal_screen: String::new(), + last_pty_cols: 0, login_shell: None, fish_config_before_install: None, + signature_requirement: SignatureRequirement::Required, }); } @@ -632,6 +710,16 @@ async fn given_login_shell_bash(world: &mut E2eWorld) { state_mut(world).login_shell = Some("/bin/bash".to_owned()); } +#[given("the user has not asked the installer to require a signature")] +async fn given_signature_requirement_unset(world: &mut E2eWorld) { + state_mut(world).signature_requirement = SignatureRequirement::Unset; +} + +#[given("the user has turned the installer's signature requirement off")] +async fn given_signature_requirement_off(world: &mut E2eWorld) { + state_mut(world).signature_requirement = SignatureRequirement::Off; +} + #[given("a signed bundle installed updating the user PATH")] async fn given_installed_updating_user_path(world: &mut E2eWorld) { package_with_key_file(world).await; @@ -994,6 +1082,26 @@ async fn when_install_no_key(world: &mut E2eWorld) { state_mut(world).last_output = out; } +#[when("a specific version of the signed bundle is installed with no public key supplied")] +async fn when_install_version_no_key(world: &mut E2eWorld) { + // Any channel other than `release` names a release tag, so the installer has + // no release-channel signature requirement to fall back on. The download + // base still points at the scenario's own packaged bundle. + let st = state(world); + let install_dir = st.install_dir.clone(); + let download_base = st.download_base.clone(); + let no_path = [("ROCM_CLI_UPDATE_SHELL_PATH", "0".to_string())]; + let (out, _ok) = run_installer_on_channel( + world, + "v0.0.0-e2e", + &install_dir, + &download_base, + &[], + &no_path, + ); + state_mut(world).last_output = out; +} + #[when("the signed bundle is installed updating the user PATH")] async fn when_install_update_user_path(world: &mut E2eWorld) { // Windows-only: a default install (no -NoPathUpdate) mutates the user PATH. @@ -2054,3 +2162,615 @@ async fn then_xdg_gone(world: &mut E2eWorld) { assert_missing(dir); } } + +/// Terminal geometry for the progress-rendering scenarios. `STALE_COLUMNS` is +/// deliberately wider than `PTY_COLS`: it stands in for a `COLUMNS` inherited +/// from a larger window, which is what a shell leaves behind when its terminal +/// is resized after it starts, and is the condition under which the +/// installer's download bar used to overrun the margin and wrap. +/// +/// `PTY_ROWS` is tall enough to hold a whole successful install, so a clean run +/// never scrolls and the rendered screen below shows every progress row at +/// once. A wrapping run overflows it, which is itself part of the symptom. +const PTY_COLS: u16 = 80; +const PTY_ROWS: u16 = 40; +const STALE_COLUMNS: u16 = 96; +/// A terminal narrow enough that curl would ignore the width handed to it, so +/// the installer declines to draw a bar at all. +const NARROW_PTY_COLS: u16 = 20; +/// Deliberately not 80. ncurses falls back to 80 columns when it cannot read a +/// terminal, so a measurement that consults the wrong file descriptor lands on +/// exactly that and is indistinguishable from a correct one at this suite's +/// usual width. At 100 the two answers differ and the test can tell them apart. +const WIDE_PTY_COLS: u16 = 100; + +/// Downloads a release install performs: the archive, its checksum sidecar and +/// its signature sidecar. One progress bar each. +const EXPECTED_DOWNLOADS: usize = 3; + +/// The shape of one pty run: how wide the terminal is, what `COLUMNS` the +/// installer inherits, and how it is started. +#[derive(Clone, Copy)] +struct PtyRun<'a> { + /// The terminal's real width. + pty_cols: u16, + /// What the installer inherits as `COLUMNS`; `None` leaves it unset. A raw + /// string rather than a number so a malformed value such as a zero-padded + /// width can be expressed. Disagreeing with `pty_cols` is the condition + /// several of these scenarios exist to test. + columns_env: Option<&'a str>, + invocation: Invocation, +} + +/// How the installer is started under the pty. +#[derive(Clone, Copy)] +enum Invocation { + /// `sh install.sh `, with the pty on every descriptor. + Direct, + /// `cat install.sh | sh -s `, mirroring the documented + /// `curl ... | sh`. The script arrives on stdin, so stdin is a pipe and + /// only stdout and stderr are still the terminal. This is the shape the + /// installer actually ships in, and the reason it measures the terminal + /// through fd 2 rather than whatever stdin happens to be. + PipedIntoShell, +} + +/// Run the installer attached to a real pseudo-terminal, returning both what it +/// wrote and what a terminal drawing those bytes would show. +/// +/// The piped `run_installer` path cannot exercise progress drawing at all: the +/// installer only draws a bar when its stderr is a terminal, so under a pipe it +/// takes the quiet branch. Only a PTY puts it on the drawing path. +/// +fn run_installer_pty( + world: &E2eWorld, + install_dir: &Path, + download_base: &str, + public_key_env: &[(&str, String)], + extra_env: &[(&str, String)], + shape: PtyRun<'_>, +) -> (String, String) { + use std::io::Read as _; + + use portable_pty::{CommandBuilder, PtySize, native_pty_system}; + + let PtyRun { + pty_cols, + columns_env, + invocation, + } = shape; + + let root = workspace_root(); + let st = state(world); + + let pair = native_pty_system() + .openpty(PtySize { + rows: PTY_ROWS, + cols: pty_cols, + pixel_width: 0, + pixel_height: 0, + }) + .expect("failed to open a pty for the installer"); + + // The scenarios carry @requires-os:linux, so only the POSIX installer is + // ever driven here. This still compiles on Windows, which builds the e2e + // target for its own lifecycle lanes, it simply never runs. + // Absolute path: a scenario may hand the child a PATH that portable-pty + // would also use to resolve this program, and `sh` must stay findable + // regardless of what that PATH shadows out. + let shell = which_on_path("sh").unwrap_or_else(|| PathBuf::from("/bin/sh")); + let script = root.join("install.sh"); + let mut cmd = CommandBuilder::new(&shell); + match invocation { + Invocation::Direct => { + cmd.arg(&script); + cmd.arg("release"); + } + Invocation::PipedIntoShell => { + cmd.arg("-c"); + // Single-quoted so a path with spaces survives; neither path can + // contain a single quote, both being repo-relative. + cmd.arg(format!( + "cat '{}' | '{}' -s release", + script.display(), + shell.display() + )); + } + } + cmd.cwd(&root); + for (key, value) in std::env::vars_os() { + cmd.env(key, value); + } + cmd.env("ROCM_CLI_DOWNLOAD_BASE", download_base); + cmd.env("ROCM_CLI_INSTALL_DIR", install_dir); + cmd.env("ROCM_CLI_REQUIRE_SIGNATURE", "1"); + cmd.env("HOME", &st.home_dir); + cmd.env("TERM", "xterm-256color"); + match columns_env { + Some(cols) => cmd.env("COLUMNS", cols), + // The harness's own COLUMNS would otherwise leak in and stand in for a + // value the scenario meant to leave unset. + None => cmd.env_remove("COLUMNS"), + } + for (key, value) in public_key_env.iter().chain(extra_env.iter()) { + cmd.env(key, value); + } + + let mut child = pair + .slave + .spawn_command(cmd) + .expect("failed to spawn the installer under a pty"); + // Drop the slave before reading: while this side stays open the master + // never reaches EOF and the read below would block past process exit. + drop(pair.slave); + + let mut reader = pair + .master + .try_clone_reader() + .expect("failed to read the installer's pty"); + let mut raw = Vec::new(); + reader + .read_to_end(&mut raw) + .expect("failed to drain the installer's pty"); + let status = child.wait().expect("installer process was not reaped"); + let output = String::from_utf8_lossy(&raw).into_owned(); + assert!( + status.success(), + "the installer failed under a pty:\n{output}" + ); + + // Draw the stream the way a terminal would. Wrapping is the emulator's + // doing and never appears in the byte stream, so this is the only view in + // which an over-wide bar actually tears. + // vt100 0.18 takes non-zero dimensions. Both are known good here: PTY_ROWS + // is a fixed constant and every scenario drives a terminal at least as wide + // as NARROW_PTY_COLS. + let rows = NonZeroU16::new(PTY_ROWS).expect("PTY_ROWS is a non-zero constant"); + let cols = NonZeroU16::new(pty_cols).expect("a pty needs a non-zero width to render into"); + let mut parser = vt100::Parser::new(rows, cols, 0); + parser.process(&raw); + // `rows()`, not `contents()`: the latter stitches a wrapped row back onto + // the one it came from, reporting a 96-column line on an 80-column screen + // and hiding the very tearing this is here to catch. + let screen = parser + .screen() + .rows(0, pty_cols) + .collect::>() + .join("\n"); + (output, screen) +} + +/// The frames the installer painted: the stretches between the carriage +/// returns that separate one redraw from the next. +/// +/// Both signals are needed, and neither is sufficient alone. A downloader +/// redraws in place and nothing else here does, but the line it redraws on can +/// also carry the preceding `download: ` notice, which is not a frame and +/// is longer than a terminal when the URL is a real release asset. Bar +/// characters alone are no better: the get-started block prints +/// `rocm install sdk # ...`, which never redraws. Requiring a carriage return +/// and a bar character keeps each one out. +fn progress_frames(terminal_output: &str) -> Vec<&str> { + terminal_output + .lines() + .filter(|line| line.contains('\r')) + .flat_map(|line| line.split('\r')) + .filter(|frame| frame.contains('#') || frame.contains('%')) + .collect() +} + +/// Rows of a rendered screen showing a drawn progress bar. +/// +/// The carriage returns are gone by this point, consumed by the emulator, so +/// this matches on shape instead: a run of bar characters, which the +/// get-started block's single `#` cannot reach and the banner art never +/// contains. +fn progress_rows(screen: &str) -> Vec<&str> { + screen + .lines() + .map(str::trim_end) + .filter(|row| row.contains("###")) + .collect() +} + +/// Serve the signed dist over loopback and install from it under a pty. +fn install_under_pty( + world: &mut E2eWorld, + pty_cols: u16, + columns_env: Option<&str>, + invocation: Invocation, +) { + let st = state(world); + let dist_dir = st.dist_dir.clone(); + let install_dir = st.install_dir.clone(); + let public_key = st.public_key.clone(); + // Loopback rather than file://, so the installer runs the HTTP download + // path that actually draws a bar. + let server = e2e_cucumber::loopback_http::LoopbackServer::start(&dist_dir); + let base = server.base_url(); + let key_env = [( + "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", + public_key.to_string_lossy().into_owned(), + )]; + let no_path = [("ROCM_CLI_UPDATE_SHELL_PATH", "0".to_string())]; + let (raw, screen) = run_installer_pty( + world, + &install_dir, + &base, + &key_env, + &no_path, + PtyRun { + pty_cols, + columns_env, + invocation, + }, + ); + let st = state_mut(world); + st.last_terminal_output = raw; + st.last_terminal_screen = screen; + st.last_pty_cols = pty_cols; + drop(server); +} + +/// Install under a pty with extra environment on top of the usual isolation. +fn install_with_env( + world: &mut E2eWorld, + pty_cols: u16, + columns_env: Option<&str>, + extra: &[(&str, String)], +) { + let st = state(world); + let dist_dir = st.dist_dir.clone(); + let install_dir = st.install_dir.clone(); + let public_key = st.public_key.clone(); + let server = e2e_cucumber::loopback_http::LoopbackServer::start(&dist_dir); + let base = server.base_url(); + let key_env = [( + "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", + public_key.to_string_lossy().into_owned(), + )]; + let mut env: Vec<(&str, String)> = vec![("ROCM_CLI_UPDATE_SHELL_PATH", "0".to_string())]; + env.extend(extra.iter().cloned()); + let (raw, screen) = run_installer_pty( + world, + &install_dir, + &base, + &key_env, + &env, + PtyRun { + pty_cols, + columns_env, + invocation: Invocation::Direct, + }, + ); + let st = state_mut(world); + st.last_terminal_output = raw; + st.last_terminal_screen = screen; + st.last_pty_cols = pty_cols; + drop(server); +} + +#[when("the signed bundle is installed through a pseudo-terminal narrower than COLUMNS")] +async fn when_install_pty_stale_columns(world: &mut E2eWorld) { + install_under_pty( + world, + PTY_COLS, + Some(&STALE_COLUMNS.to_string()), + Invocation::Direct, + ); +} + +#[when("the signed bundle is installed through a terminal too narrow for a progress bar")] +async fn when_install_pty_narrow(world: &mut E2eWorld) { + install_under_pty(world, NARROW_PTY_COLS, None, Invocation::Direct); +} + +#[when("the signed bundle is installed with a zero-padded COLUMNS and no way to measure")] +async fn when_install_pty_zero_padded_columns(world: &mut E2eWorld) { + // COLUMNS is the last source the width falls back to, so it is only + // consulted when neither stty nor tput can answer. With a real pty, stty + // answers first and this branch is unreachable: hide both tools. + // + // "080" passes a digits-only check but is an octal literal to POSIX + // arithmetic, and dash aborts the whole script on it. Only a raw string can + // express that, which is why columns_env is not a number. + let path = shadow_path(world, &["stty", "tput"], &[]); + install_with_env(world, PTY_COLS, Some("080"), &[("PATH", path)]); +} + +#[then("every download progress frame stays within the terminal")] +async fn then_progress_fits_terminal(world: &mut E2eWorld) { + let output = state(world).last_terminal_output.clone(); + let frames = progress_frames(&output); + assert!( + !frames.is_empty(), + "the installer drew no progress at all under a pty:\n{output}" + ); + let width = usize::from(state(world).last_pty_cols); + // A frame wider than the terminal certainly wraps, and one exactly as wide + // as it sits on the right margin, where terminals disagree about whether + // the next write moves down a row. Either way the carriage return ending + // the frame can land on a row below the bar's, so every later redraw paints + // one row lower. Reject both. + let overflowing: Vec<&&str> = frames + .iter() + .filter(|frame| frame.chars().count() >= width) + .collect(); + assert!( + overflowing.is_empty(), + "{} of {} progress frames reach or exceed the {width}-column terminal, \ + so each redraw wraps onto a new row instead of replacing the last; \ + widest was {} columns:\n{output}", + overflowing.len(), + frames.len(), + overflowing + .iter() + .map(|frame| frame.chars().count()) + .max() + .unwrap_or(0), + ); +} + +#[then("the finished screen shows one completed bar per download")] +async fn then_one_bar_per_download(world: &mut E2eWorld) { + // Asserted against the rendered screen, not the byte stream. A bar that + // overruns the margin is torn in two by the terminal, which leaves the + // leftovers of every earlier redraw stranded on their own rows; none of + // that exists in the bytes, where each redraw is just one over-wide frame. + let screen = state(world).last_terminal_screen.clone(); + let rows = progress_rows(&screen); + assert_eq!( + rows.len(), + EXPECTED_DOWNLOADS, + "expected {EXPECTED_DOWNLOADS} progress rows on the finished screen, \ + one per download, but found {}. More than that means each redraw left \ + the previous bar behind instead of replacing it:\n{screen}", + rows.len(), + ); + // Every surviving row is a finished bar. A stack of wrapped redraws leaves + // partial bars with no percentage, and a torn one leaves a stranded tail. + let unfinished: Vec<&&str> = rows.iter().filter(|row| !row.contains("100.0%")).collect(); + assert!( + unfinished.is_empty(), + "{} of {} progress rows do not carry a whole 100.0%, so a bar was left \ + partially drawn or its percentage was torn across the margin: {:?}\n{screen}", + unfinished.len(), + rows.len(), + unfinished, + ); +} + +#[then("the installer reports that the terminal is too narrow for a progress bar")] +async fn then_narrow_notice(world: &mut E2eWorld) { + // The notice is longer than this terminal is wide, so on the rendered + // screen it is split across physical rows and no single row contains it. + // Read it from the byte stream, which carries it whole, and keep the + // screen for the question it alone can answer: was a bar drawn. + let output = state(world).last_terminal_output.clone(); + assert!( + output.contains("too narrow for a progress bar"), + "a terminal too narrow to size a bar downloads quietly, which should \ + say so rather than look like a stall:\n{output}" + ); + let screen = state(world).last_terminal_screen.clone(); + let rows = progress_rows(&screen); + assert!( + rows.is_empty(), + "no progress bar should be drawn at {NARROW_PTY_COLS} columns, but the \ + screen carries {}: {rows:?}\n{screen}", + rows.len(), + ); +} + +/// A single directory that shadows `PATH`: a symlink to every executable +/// reachable on it, minus `remove`, plus a shim script for each `shims` entry. +/// +/// Dropping whole directories cannot work here: the tools these scenarios need +/// out of the way live in `/usr/bin` beside `sh`, `tar`, `awk` and ~980 others +/// the installer and harness both need, so removing that directory leaves +/// nothing able to run. Shadowing individual names keeps everything else where +/// the installer expects it, and `command -v ` then genuinely misses. +/// +/// Shims exist because some branches are only reachable when a tool answers a +/// particular way: a terminal that never had its size set reports `0 0`, which +/// no real pty here will do on demand. +fn shadow_path(world: &E2eWorld, remove: &[&str], shims: &[(&str, &str)]) -> String { + let tag: String = remove + .iter() + .chain(shims.iter().map(|(name, _)| name)) + .copied() + .collect::>() + .join("-"); + let farm = root(world).join("shadow-path").join(if tag.is_empty() { + "none".to_owned() + } else { + tag + }); + std::fs::create_dir_all(&farm).expect("failed to create the shadow PATH directory"); + let hidden: Vec<&str> = remove + .iter() + .chain(shims.iter().map(|(name, _)| name)) + .copied() + .collect(); + for dir in std::env::var("PATH").unwrap_or_default().split(':') { + if dir.is_empty() { + continue; + } + let Ok(entries) = std::fs::read_dir(dir) else { + continue; + }; + for entry in entries.flatten() { + let file_name = entry.file_name(); + if hidden.iter().any(|name| file_name == *name) { + continue; + } + let link = farm.join(&file_name); + // First writer wins, so earlier PATH entries keep their precedence. + if !link.exists() { + #[cfg(unix)] + { + let _ = std::os::unix::fs::symlink(entry.path(), &link); + } + // Only the @requires-os:linux progress scenarios build a shadow + // PATH, so this arm never runs. It exists because the Windows + // lifecycle lanes still compile this target, and a symlink is + // the one piece here with no portable spelling. + #[cfg(not(unix))] + { + unreachable!("a shadow PATH is only built on Unix"); + } + } + } + } + for (name, body) in shims { + let path = farm.join(name); + let _ = std::fs::remove_file(&path); + std::fs::write(&path, body).expect("failed to write a PATH shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .expect("failed to make a PATH shim executable"); + } + } + for name in remove { + assert!( + !farm.join(name).exists(), + "{name} is still resolvable through the shadow PATH" + ); + } + farm.to_string_lossy().into_owned() +} + +#[when("the signed bundle is installed through a pseudo-terminal with curl unavailable")] +async fn when_install_pty_without_curl(world: &mut E2eWorld) { + assert!( + which_on_path("wget").is_some(), + "this scenario covers the installer's wget branch and the host has no \ + wget; install it or the branch goes untested" + ); + let st = state(world); + let dist_dir = st.dist_dir.clone(); + let install_dir = st.install_dir.clone(); + let public_key = st.public_key.clone(); + let server = e2e_cucumber::loopback_http::LoopbackServer::start(&dist_dir); + let base = server.base_url(); + let key_env = [( + "ROCM_CLI_SIGNING_PUBLIC_KEY_PATH", + public_key.to_string_lossy().into_owned(), + )]; + let extra = [ + ("ROCM_CLI_UPDATE_SHELL_PATH", "0".to_string()), + ("PATH", shadow_path(world, &["curl"], &[])), + ]; + let (raw, screen) = run_installer_pty( + world, + &install_dir, + &base, + &key_env, + &extra, + PtyRun { + pty_cols: PTY_COLS, + columns_env: Some(&STALE_COLUMNS.to_string()), + invocation: Invocation::Direct, + }, + ); + let st = state_mut(world); + st.last_terminal_output = raw; + st.last_terminal_screen = screen; + st.last_pty_cols = PTY_COLS; + drop(server); +} + +/// First directory on `PATH` holding `name`, or `None`. +/// +/// Splits on `:`, so this is Unix-shaped. Every caller sits behind a +/// `@requires-os:linux` scenario; it compiles on Windows but would not find +/// anything there. +fn which_on_path(name: &str) -> Option { + std::env::var("PATH").ok().and_then(|path| { + path.split(':') + .map(|dir| Path::new(dir).join(name)) + .find(|candidate| candidate.exists()) + }) +} + +#[then("the download prints no request logging around the bar")] +async fn then_no_request_logging(world: &mut E2eWorld) { + let output = state(world).last_terminal_output.clone(); + // wget's default verbosity narrates every step of the request, which lands + // interleaved with the progress bar. These lines are what -nv drops while + // still leaving the bar, and still reporting a failure's reason. + for noise in ["Connecting to", "HTTP request sent", "Length:"] { + assert!( + !output.contains(noise), + "{noise:?} appears around the progress bar, so the downloader is \ + narrating the request instead of just drawing the bar:\n{output}" + ); + } +} + +#[when("the signed bundle is installed by piping the installer into a shell")] +async fn when_install_pty_piped(world: &mut E2eWorld) { + install_under_pty(world, WIDE_PTY_COLS, None, Invocation::PipedIntoShell); +} + +#[when("the signed bundle is installed where the terminal reports no size")] +async fn when_install_pty_unsized(world: &mut E2eWorld) { + // A pty that never had its window size set reports `0 0`. No real pty here + // will do that on demand, so stand in a stty that says so. + // + // tput is deliberately left available, and the terminal is deliberately not + // 80 columns wide. Treating the 0 as a width suppresses the bar, but a + // later guard would then rescue it to the 80-column default, which at this + // suite's usual width is indistinguishable from success. Falling through to + // tput instead yields the terminal's real width, so the two outcomes differ + // and the assertion can tell them apart. + let path = shadow_path(world, &[], &[("stty", "#!/bin/sh\nprintf '0 0\\n'\n")]); + install_with_env(world, WIDE_PTY_COLS, None, &[("PATH", path)]); +} + +#[then("each completed bar spans the measured terminal width")] +async fn then_bar_spans_terminal(world: &mut E2eWorld) { + let st = state(world); + let screen = st.last_terminal_screen.clone(); + let cols = usize::from(st.last_pty_cols); + let expected = cols - 1; + let rows = progress_rows(&screen); + assert!( + !rows.is_empty(), + "the installer drew no progress bar at all:\n{screen}" + ); + // The width is the assertion. A bar drawn to some other width still fits + // the terminal and still redraws in place, so the width check alone cannot + // see it, yet it means the measurement came from somewhere other than this + // terminal: ncurses answers 80 when it is asked about a pipe rather than + // the real descriptor. + let wrong: Vec<(usize, &&str)> = rows + .iter() + .map(|row| (row.chars().count(), row)) + .filter(|(len, _)| *len != expected) + .collect(); + assert!( + wrong.is_empty(), + "a completed bar should span {expected} of this terminal's {cols} \ + columns, reserving only the last; got widths {:?}. A narrower bar \ + means the width was not read from this terminal:\n{screen}", + wrong.iter().map(|(len, _)| *len).collect::>(), + ); +} + +#[then("the installer still draws a progress bar")] +async fn then_bar_still_drawn(world: &mut E2eWorld) { + // Asserted on the byte stream, not a rendered screen: a pty that never had + // a size cannot be drawn into meaningfully, and the question here is only + // whether the installer gave up on progress or not. + let output = state(world).last_terminal_output.clone(); + let frames = progress_frames(&output); + assert!( + !frames.is_empty(), + "an unsized terminal reports 0 columns, which means unmeasured rather \ + than unusably narrow; the installer should still draw a bar at a \ + fallback width instead of downloading in silence:\n{output}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs index 1838780c1e..429d3573ca 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs @@ -27,6 +27,14 @@ use crate::E2eWorld; // `:` form only in `runtime_id`, which is a manifest field value, never a filename. const FIRST_KEY: &str = "release-tarball-gfx942"; const SECOND_KEY: &str = "release-tarball-gfx1100"; +/// The GPU families `FIRST_KEY` and `SECOND_KEY` are planted with; each family +/// also names the runtime's install folder (see [`planted_install_root`]). +const FIRST_FAMILY: &str = "gfx942"; +/// The version every planted runtime records: a dated nightly-style version, so +/// the JSON (which carries the version as recorded) and the text (which adds +/// the build date) can be told apart. +const PLANTED_VERSION: &str = "7.14.0a20260601"; +const SECOND_FAMILY: &str = "gfx1100"; /// Two keys that differ only in letter case. Each is its own `.json` in the /// registry, so the pair can only coexist on a case-sensitive filesystem — which /// is why the scenario using them carries `@requires-case-sensitive-fs`. @@ -66,8 +74,7 @@ fn plant_runtime(world: &E2eWorld, key: &str, family: &str) -> PathBuf { .all(|ch| ch.is_ascii_alphanumeric() || ch == '-'), "runtime key must be production-style and safe as a registry filename: {key}" ); - let root = world.isolated_root.as_ref().expect("no isolated root"); - let install_root = root.path().join(format!("runtime-{family}")); + let install_root = planted_install_root(world, family); std::fs::create_dir_all(&install_root).expect("failed to create install root"); std::fs::write(install_root.join("payload.txt"), "payload") .expect("failed to write runtime payload"); @@ -80,10 +87,21 @@ fn plant_runtime(world: &E2eWorld, key: &str, family: &str) -> PathBuf { install_root } +/// Where [`plant_runtime`] puts the install root of a runtime of `family`. +fn planted_install_root(world: &E2eWorld, family: &str) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join(format!("runtime-{family}")) +} + /// The runtime registry under the scenario's isolated data dir — where the CLI /// reads `.json` manifests from. One definition, so a layout change cannot -/// leave the planting and the counting looking in different places. -fn registry_dir(world: &E2eWorld) -> PathBuf { +/// leave the planting and the counting looking in different places; the +/// side-by-side steps read it from here too. +pub(crate) fn registry_dir(world: &E2eWorld) -> PathBuf { world .isolated_root .as_ref() @@ -118,7 +136,7 @@ fn runtime_manifest_json(key: &str, family: &str, install_root: &Path) -> String "format": "tarball", "family": family, "family_source": "manual", - "version": "1.0.0", + "version": PLANTED_VERSION, "install_root": install_root, "selected_artifact_url": format!("https://example.invalid/{key}.tar.gz"), "read_only": true, @@ -127,6 +145,114 @@ fn runtime_manifest_json(key: &str, family: &str, install_root: &Path) -> String .expect("failed to serialize runtime manifest") } +/// A nightly ROCm 10 wheel runtime installed for one exact GPU. Its family is the +/// family-wide `gfx120X-all` while its exact target is `gfx1201`, so a listing +/// that guessed the target from the family would show a different value. +pub(crate) const NIGHTLY_EXACT_TARGET_KEY: &str = "nightly-wheel-gfx1201-10-1-0a20260822"; +/// The package index the nightly runtime records. `.invalid` never resolves; +/// nothing here is fetched, the value is only read back. +pub(crate) const NIGHTLY_SOURCE: &str = "https://example.invalid/whl-multi-arch"; +/// The artifact URL the nightly runtime records beside its index. Deliberately +/// different from [`NIGHTLY_SOURCE`], so a listing that showed the artifact URL +/// instead of the package index would be caught. (`rocm install sdk` writes the +/// index into both; an imported manifest can carry two different values.) +const NIGHTLY_ARTIFACT_URL: &str = "https://example.invalid/artifacts/rocm-10.1.0a20260822"; +/// The source layout generation a ROCm 10 nightly is published under. +pub(crate) const NIGHTLY_SOURCE_LAYOUT: &str = "next-v1"; +/// The exact GPU target the nightly runtime's packages were installed for. +pub(crate) const NIGHTLY_DEVICE_TARGET: &str = "gfx1201"; + +/// Write a read-only nightly `wheel` runtime manifest recording every provenance +/// fact the way `rocm install sdk` records it: channel, package index, source +/// layout generation, and the exact GPU target in its wheel composition. Its +/// version embeds a build date. Read-only and with no Python, it lists as +/// unusable — which does not matter to what these scenarios read from it. +pub(crate) fn plant_nightly_exact_target_runtime(world: &E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let install_root = root.path().join("runtime-nightly-gfx1201"); + std::fs::create_dir_all(&install_root).expect("failed to create install root"); + let registry = registry_dir(world); + std::fs::create_dir_all(®istry).expect("failed to create registry dir"); + let manifest = serde_json::json!({ + "runtime_key": NIGHTLY_EXACT_TARGET_KEY, + "runtime_id": "therock-nightly:gfx120X-all", + "channel": "nightly", + "format": "wheel", + "family": "gfx120X-all", + "family_source": "detected", + "version": "10.1.0a20260822", + "install_root": install_root, + "selected_artifact_url": NIGHTLY_ARTIFACT_URL, + "source_layout_generation": NIGHTLY_SOURCE_LAYOUT, + "index_url": NIGHTLY_SOURCE, + "wheel_composition": { + "source_layout_generation": NIGHTLY_SOURCE_LAYOUT, + "package_specs": [format!("rocm[libraries,device-{NIGHTLY_DEVICE_TARGET}]==10.1.0a20260822")], + "rocm_sdk_target": NIGHTLY_DEVICE_TARGET, + }, + "read_only": true, + "devel": false, + "installed_at_unix_ms": 1_700_000_000_000u64, + }); + std::fs::write( + registry.join(format!("{NIGHTLY_EXACT_TARGET_KEY}.json")), + serde_json::to_string_pretty(&manifest).expect("failed to serialize runtime manifest"), + ) + .expect("failed to write runtime manifest"); +} + +/// Key of the planted adopted runtime, in the shape `rocm runtimes adopt` +/// derives when no key is given: `adopted-{channel}-pip-{family}-{version}`. +const ADOPTED_KEY: &str = "adopted-release-pip-gfx1100-7-13-0"; +/// Version matching [`ADOPTED_KEY`]. +const ADOPTED_VERSION: &str = "7.13.0"; + +/// Write a runtime manifest carrying the fields the listing reads, as adoption +/// writes them for an environment ROCm CLI did not install: read-only, +/// `imported_from` its own folder, no package index or source layout, and the +/// adoption marker where an artifact URL would be. It is not everything +/// `rocm runtimes adopt` writes (there is no SDK probe or Python), and it is +/// planted rather than adopted because adopting needs a real ROCm SDK +/// environment to probe. +fn plant_adopted_runtime(world: &E2eWorld) { + let install_root = plant_runtime(world, ADOPTED_KEY, "gfx1100"); + let manifest_path = registry_dir(world).join(format!("{ADOPTED_KEY}.json")); + let mut manifest: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&manifest_path).expect("failed to read planted manifest"), + ) + .expect("planted manifest is not JSON"); + let fields = manifest + .as_object_mut() + .expect("planted manifest is not an object"); + fields.insert("version".into(), ADOPTED_VERSION.into()); + fields.insert("format".into(), "wheel".into()); + fields.insert("family_source".into(), "runtime_id".into()); + fields.insert("imported_from".into(), serde_json::json!(install_root)); + // The literal adoption has always written here, NOT the CLI's constant: + // runtimes adopted by earlier releases hold exactly this on disk, so the + // listing must keep recognising it. If the CLI changes the marker without + // a migration, this scenario goes red, as does the unit test + // `adoption_marker_keeps_its_persisted_value` in runtime_registry.rs, + // which pins the same literal against the constant. + fields.insert("selected_artifact_url".into(), "adopted-read-only".into()); + std::fs::write( + &manifest_path, + serde_json::to_string_pretty(&manifest).expect("failed to serialize runtime manifest"), + ) + .expect("failed to write runtime manifest"); +} + +/// The `provenance:` line the listing prints under `runtime_key`'s row: the +/// first one after the row that names the key. +fn listed_provenance<'a>(out: &'a str, runtime_key: &str) -> &'a str { + let row = format!(" {runtime_key} runtime_id="); + out.lines() + .skip_while(|line| !line.contains(&row)) + .skip(1) + .find_map(|line| line.trim().strip_prefix("provenance: ")) + .unwrap_or_else(|| panic!("no provenance line listed for {runtime_key}:\n{out}")) +} + /// Plant a managed-service record the CLI reads as a LIVE local server that was /// launched against `recorded_runtime_key`, so activation has something real to /// report on. Uses the shared record schema (`write_service_record_with`) rather @@ -222,6 +348,38 @@ fn ids_listed_under<'a>(report: &'a str, section: &str) -> Vec<&'a str> { .collect() } +/// The isolated `config.json`, where the CLI records the active runtime. +pub(crate) fn config_path(world: &E2eWorld) -> PathBuf { + world.app_paths().config_dir.join("config.json") +} + +/// The isolated active-runtime marker (`/runtimes/active.json`) — what +/// the next serve would actually use. +pub(crate) fn active_marker_path(world: &E2eWorld) -> PathBuf { + world + .app_paths() + .data_dir + .join("runtimes") + .join("active.json") +} + +/// A JSON file the CLI writes, parsed, or `None` when there is no such file — +/// which is how the CLI leaves the config and the marker before anything was +/// activated, and the marker after the runtime in use was uninstalled. Any +/// other read failure, and a file that is not JSON, fail the step. +pub(crate) fn read_json_if_present(path: &Path) -> Option { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => panic!("failed to read {}: {error}", path.display()), + }; + Some( + serde_json::from_str(&text).unwrap_or_else(|error| { + panic!("{} is not valid JSON: {error}\n{text}", path.display()) + }), + ) +} + /// The runtime key recorded in the isolated `config.json`. /// /// The marker (see [`active_runtime_key`]) cannot be used by the scenario that @@ -229,45 +387,45 @@ fn ids_listed_under<'a>(report: &'a str, section: &str) -> Vec<&'a str> { /// anyway: it is written first, so it is the half that would be left naming the /// new runtime if the restore did not run. fn config_active_runtime_key(world: &E2eWorld) -> Option { - let config_dir = world - .isolate_env() - .into_iter() - .find(|(key, _)| *key == "ROCM_CLI_CONFIG_DIR") - .map(|(_, value)| PathBuf::from(value)) - .expect("isolate_env did not set ROCM_CLI_CONFIG_DIR"); - let config = config_dir.join("config.json"); - let text = std::fs::read_to_string(&config) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", config.display())); - let value: serde_json::Value = serde_json::from_str(&text) - .unwrap_or_else(|error| panic!("{} is not valid JSON: {error}", config.display())); - value + read_config(world) .get("active_runtime_key") .and_then(serde_json::Value::as_str) .map(str::to_owned) } -/// The runtime key recorded in the isolated active-runtime marker -/// (`/runtimes/active.json`) — what the next serve would actually use. +/// The isolated `config.json`, parsed; a missing file fails the step. +fn read_config(world: &E2eWorld) -> serde_json::Value { + let config = config_path(world); + read_json_if_present(&config) + .unwrap_or_else(|| panic!("failed to read {}: it does not exist", config.display())) +} + +/// Edit the config the way a person editing the file by hand would: set +/// `active_runtime_key` and leave every other value as the CLI wrote it. +fn set_config_active_runtime_key(world: &E2eWorld, runtime_key: &str) { + let mut config = read_config(world); + config["active_runtime_key"] = serde_json::Value::from(runtime_key); + let path = config_path(world); + std::fs::write( + &path, + serde_json::to_string_pretty(&config).expect("config serializes"), + ) + .unwrap_or_else(|error| panic!("failed to write {}: {error}", path.display())); +} + +/// The runtime key recorded in the isolated active-runtime marker. /// /// Parsed as JSON rather than substring-matched on the file: once a second /// runtime has been activated the marker also carries `previous_runtime_key`, /// so a `contains` check would be satisfied by either key and could not tell a /// refused switch from a completed one. fn active_runtime_key(world: &E2eWorld) -> String { - let root = world.isolated_root.as_ref().expect("no isolated root"); - let marker = root - .path() - .join("data") - .join("runtimes") - .join("active.json"); - let text = std::fs::read_to_string(&marker) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", marker.display())); - let value: serde_json::Value = serde_json::from_str(&text) - .unwrap_or_else(|error| panic!("{} is not valid JSON: {error}", marker.display())); - value + let marker = active_marker_path(world); + read_json_if_present(&marker) + .unwrap_or_else(|| panic!("failed to read {}: it does not exist", marker.display())) .get("runtime_key") .and_then(serde_json::Value::as_str) - .unwrap_or_else(|| panic!("no runtime_key in {}:\n{text}", marker.display())) + .unwrap_or_else(|| panic!("no runtime_key in {}", marker.display())) .to_owned() } @@ -302,20 +460,80 @@ fn write_import_manifest(world: &E2eWorld) -> PathBuf { #[given("two registered runtimes and none active")] async fn two_runtimes(world: &mut E2eWorld) { - plant_runtime(world, FIRST_KEY, "gfx942"); - plant_runtime(world, SECOND_KEY, "gfx1100"); + plant_runtime(world, FIRST_KEY, FIRST_FAMILY); + plant_runtime(world, SECOND_KEY, SECOND_FAMILY); } #[given("two registered runtimes with the second active after the first")] async fn two_runtimes_second_active(world: &mut E2eWorld) { - plant_runtime(world, FIRST_KEY, "gfx942"); - plant_runtime(world, SECOND_KEY, "gfx1100"); + plant_runtime(world, FIRST_KEY, FIRST_FAMILY); + plant_runtime(world, SECOND_KEY, SECOND_FAMILY); // Activate first, then second, so `previous_runtime_key` records the first — // the state rollback must return to. crate::run_rocm_ok(world, &["runtimes", "activate", FIRST_KEY]); crate::run_rocm_ok(world, &["runtimes", "activate", SECOND_KEY]); } +/// Nothing to plant: each scenario starts from an empty isolated root. Asserted +/// rather than assumed, so a fixture that starts seeding runtimes cannot turn +/// the empty-registry scenario into one that lists something. +#[given("no ROCm runtime is registered")] +async fn no_runtime_registered(world: &mut E2eWorld) { + let registry = registry_dir(world); + let registered = std::fs::read_dir(®istry).map_or(0, Iterator::count); + assert_eq!( + registered, + 0, + "the isolated registry {} must start empty", + registry.display() + ); +} + +#[given("the first runtime's folder has been deleted")] +async fn first_runtime_folder_deleted(world: &mut E2eWorld) { + std::fs::remove_dir_all(planted_install_root(world, FIRST_FAMILY)) + .expect("failed to delete the first runtime's install root"); +} + +#[given("the second runtime's registry entry has been deleted")] +async fn second_runtime_manifest_deleted(world: &mut E2eWorld) { + std::fs::remove_file(registry_dir(world).join(format!("{SECOND_KEY}.json"))) + .expect("failed to delete the second runtime's manifest"); +} + +/// The registry entry is replaced by a folder of the same name: the listing +/// finds it among the entries and fails reading it, on every platform and +/// whoever runs the suite, where a file with its permissions removed is still +/// read by root and cannot be made unreadable the same way on Windows. An entry +/// that reads but does not parse would not do: the listing skips those. +#[given("the first runtime's registry entry cannot be read")] +async fn first_runtime_manifest_unreadable(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{FIRST_KEY}.json")); + std::fs::remove_file(&entry) + .unwrap_or_else(|error| panic!("failed to delete {}: {error}", entry.display())); + std::fs::create_dir(&entry) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", entry.display())); +} + +/// A key no runtime is registered under: what the config holds after the +/// active runtime's registry entry goes without the CLI noticing. +const UNREGISTERED_KEY: &str = "release-tarball-gfx90a"; + +#[given("the configuration names the first runtime as active in other letter case")] +async fn config_names_first_in_other_case(world: &mut E2eWorld) { + let other_case = FIRST_KEY.to_ascii_uppercase(); + assert_ne!( + other_case, FIRST_KEY, + "the edit must change the letter case" + ); + set_config_active_runtime_key(world, &other_case); +} + +#[given("the configuration names an unregistered runtime as active")] +async fn config_names_unregistered_runtime(world: &mut E2eWorld) { + set_config_active_runtime_key(world, UNREGISTERED_KEY); +} + #[given("two registered runtimes whose keys differ only in letter case")] async fn two_case_twin_runtimes(world: &mut E2eWorld) { // Distinct families only so each twin gets its own install root. @@ -347,8 +565,8 @@ async fn two_case_twin_runtimes(world: &mut E2eWorld) { #[given("two registered runtimes and a local server recorded on the first")] async fn two_runtimes_and_service_on_first(world: &mut E2eWorld) { - plant_runtime(world, FIRST_KEY, "gfx942"); - plant_runtime(world, SECOND_KEY, "gfx1100"); + plant_runtime(world, FIRST_KEY, FIRST_FAMILY); + plant_runtime(world, SECOND_KEY, SECOND_FAMILY); // Planted BEFORE either activation, so the same record covers both halves of // the scenario: while the first runtime is the one being activated the // server is on it (nothing left behind), and activating the second leaves it @@ -358,8 +576,8 @@ async fn two_runtimes_and_service_on_first(world: &mut E2eWorld) { #[given("two registered runtimes and a publicly bound local server recorded on the first")] async fn two_runtimes_and_public_service_on_first(world: &mut E2eWorld) { - plant_runtime(world, FIRST_KEY, "gfx942"); - plant_runtime(world, SECOND_KEY, "gfx1100"); + plant_runtime(world, FIRST_KEY, FIRST_FAMILY); + plant_runtime(world, SECOND_KEY, SECOND_FAMILY); // `0.0.0.0` with no endpoint key file planted is the whole setup. Restarting // a public bind that has no key is refused before the stop, so the // activation's restart fails while the server is left up on the first @@ -369,8 +587,8 @@ async fn two_runtimes_and_public_service_on_first(world: &mut E2eWorld) { #[given("two registered runtimes and a local server recording no runtime")] async fn two_runtimes_and_service_without_runtime(world: &mut E2eWorld) { - plant_runtime(world, FIRST_KEY, "gfx942"); - plant_runtime(world, SECOND_KEY, "gfx1100"); + plant_runtime(world, FIRST_KEY, FIRST_FAMILY); + plant_runtime(world, SECOND_KEY, SECOND_FAMILY); // `runtime_id: None` is the whole setup — a record that predates runtime // pinning and so says nothing about which runtime its server loaded. write_service_record_with( @@ -397,12 +615,7 @@ async fn two_runtimes_and_service_without_runtime(world: &mut E2eWorld) { /// written normally. #[given("the active runtime marker cannot be written")] async fn marker_cannot_be_written(world: &mut E2eWorld) { - let root = world.isolated_root.as_ref().expect("no isolated root"); - let marker = root - .path() - .join("data") - .join("runtimes") - .join("active.json"); + let marker = active_marker_path(world); std::fs::remove_file(&marker) .unwrap_or_else(|error| panic!("failed to remove {}: {error}", marker.display())); std::fs::create_dir_all(&marker) @@ -432,11 +645,48 @@ async fn services_cannot_be_read(world: &mut E2eWorld) { #[given("a registered read-only runtime")] async fn one_readonly_runtime(world: &mut E2eWorld) { - let install_root = plant_runtime(world, FIRST_KEY, "gfx942"); + let install_root = plant_runtime(world, FIRST_KEY, FIRST_FAMILY); // Stash the install_root path so the uninstall scenario can assert it survives. world.model_name = Some(install_root.to_string_lossy().into_owned()); } +#[given("a registered nightly runtime installed for one exact GPU")] +async fn registered_nightly_exact_target_runtime(world: &mut E2eWorld) { + plant_nightly_exact_target_runtime(world); +} + +/// The planted tarball manifest records its archive URL and nothing about a +/// source layout or an exact GPU target, as manifests written before those +/// fields did. Its family is the exact `gfx942`, so a target guessed from the +/// family would be indistinguishable from a recorded one — which is the guess +/// the listing must not make. +#[given("a registered runtime installed before its source layout and GPU target were recorded")] +async fn registered_runtime_without_recorded_provenance(world: &mut E2eWorld) { + plant_runtime(world, FIRST_KEY, "gfx942"); +} + +#[given("a registered runtime adopted from an environment ROCm CLI did not install")] +async fn registered_adopted_runtime(world: &mut E2eWorld) { + plant_adopted_runtime(world); +} + +/// A registered wheel runtime makes `rocm` start with the startup update check, +/// which looks for a usable Python and the network, and prints a progress line +/// on stdout ahead of a `--json` document when the Python on `PATH` does not fit +/// (the examine steps turn it off for the same reason). The scenarios that set +/// this are about the listing, not updates. Like every scenario env it is +/// consumed by one run, so a step that runs `rocm` again sets it again. +fn disable_startup_update_check(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1".into())); +} + +#[given("ROCm CLI does not check for runtime updates when it starts")] +async fn no_startup_update_check(world: &mut E2eWorld) { + disable_startup_update_check(world); +} + #[given("a runtime manifest to import")] async fn manifest_to_import(world: &mut E2eWorld) { let path = write_import_manifest(world); @@ -584,6 +834,27 @@ async fn list_runtimes(world: &mut E2eWorld) { record(world, stdout, stderr, rc); } +/// The first runtime's id in upper case, which differs from the registered +/// `therock-release:gfx942` only in letter case. +fn first_runtime_id_in_other_case() -> String { + format!("therock-release:{FIRST_FAMILY}").to_ascii_uppercase() +} + +#[when("the user sets the default runtime to the first runtime's id in other letter case")] +async fn set_default_runtime_in_other_case(world: &mut E2eWorld) { + let runtime_id = first_runtime_id_in_other_case(); + crate::run_rocm_ok(world, &["config", "set-default-runtime", &runtime_id]); +} + +/// Runs with the scenario's command environment (none unless a Given step set +/// one), which this run consumes. +#[when("the user lists the registered runtimes as JSON")] +async fn list_runtimes_json(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["runtimes", "list", "--json"]); + record(world, stdout, stderr, rc); +} + #[when("the user uninstalls that runtime")] async fn uninstall(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -872,6 +1143,399 @@ async fn second_marked_rollback(world: &mut E2eWorld) { ); } +/// The whole of stdout parsed as one JSON value: `serde_json::from_str` refuses +/// anything but whitespace after the value, so a stray line before or after the +/// document fails here rather than in a script that pipes it. +fn listing_json(world: &E2eWorld) -> serde_json::Value { + let out = ok_output(world); + serde_json::from_str(&out).unwrap_or_else(|error| { + panic!( + "stdout of `runtimes list --json` is not one JSON document ({error}):\n{}", + combined(world) + ) + }) +} + +/// The `runtime_key` of every listed runtime whose boolean `flag` is set. +/// Every runtime must carry `flag` as a boolean: read through `runtime[flag]` +/// alone, a missing or renamed field would count as `false` and an assertion +/// that nothing is flagged would still pass. +fn listed_keys_with(listing: &serde_json::Value, flag: &str) -> Vec { + listing["runtimes"] + .as_array() + .unwrap_or_else(|| panic!("`runtimes` is not an array:\n{listing:#}")) + .iter() + .filter(|runtime| { + runtime + .get(flag) + .and_then(serde_json::Value::as_bool) + .unwrap_or_else(|| panic!("a runtime has no boolean `{flag}`:\n{listing:#}")) + }) + .map(|runtime| { + runtime["runtime_key"] + .as_str() + .unwrap_or_else(|| panic!("a runtime has no runtime_key:\n{listing:#}")) + .to_owned() + }) + .collect() +} + +/// The `runtime_key`s of the runtime rows the text form of `runtimes list` +/// marks with `glyph`, in listing order. A row is exactly what the CLI's +/// `write_marked_row` prints under `installed:`: two spaces, the marker, a +/// space, then ` runtime_id=...`. Anchoring on that layout keeps a +/// legend, status or path line that happens to start with `- ` from counting. +fn text_rows_marked(text: &str, glyph: &str) -> Vec { + text.lines() + .filter_map(|line| line.strip_prefix(&format!(" {glyph} "))) + .filter_map(|row| { + let (key, rest) = row.split_once(' ')?; + rest.starts_with("runtime_id=").then(|| key.to_owned()) + }) + .collect() +} + +#[then("the output is a single JSON document and nothing else")] +async fn output_is_one_json_document(world: &mut E2eWorld) { + let listing = listing_json(world); + assert!( + listing.is_object(), + "expected a JSON object, got:\n{listing:#}" + ); + let out = world.cli_output.as_deref().unwrap_or_default(); + assert!( + out.starts_with('{'), + "stdout must start with the document itself, got {out:?}" + ); + assert!( + out.ends_with('\n') && !out.ends_with("\n\n"), + "the document must end with exactly one newline, got {out:?}" + ); +} + +/// `key` is present in `value` and `null`. `value[key]` alone reads a missing +/// key as `null` too, so it would not notice the key being dropped. +fn assert_null_field(value: &serde_json::Value, key: &str) { + assert_eq!( + value.as_object().and_then(|object| object.get(key)), + Some(&serde_json::Value::Null), + "`{key}` must be present and null:\n{value:#}" + ); +} + +/// The listed runtime whose `runtime_key` is `key`. +fn listed_runtime<'a>(listing: &'a serde_json::Value, key: &str) -> &'a serde_json::Value { + listing["runtimes"] + .as_array() + .unwrap_or_else(|| panic!("`runtimes` is not an array:\n{listing:#}")) + .iter() + .find(|runtime| runtime["runtime_key"] == key) + .unwrap_or_else(|| panic!("{key} is not listed:\n{listing:#}")) +} + +/// Both planted runtimes, each with the values `plant_runtime` recorded in its +/// manifest, and nothing else. The listing's order is not checked here: both +/// share one install time, so the order rests on the `runtime_key` tie-break, +/// which the unit tests +/// `runtimes_list_json_has_exactly_the_documented_fields_newest_install_first` +/// and `registry_sort_is_newest_install_first_then_by_key` pin. +#[then("the JSON lists both runtimes with their recorded values")] +async fn json_lists_both_runtimes(world: &mut E2eWorld) { + let listing = listing_json(world); + let mut listed: Vec<&str> = listing["runtimes"] + .as_array() + .expect("`runtimes` is an array") + .iter() + .map(|runtime| runtime["runtime_key"].as_str().unwrap_or_default()) + .collect(); + listed.sort_unstable(); + let mut expected = [FIRST_KEY, SECOND_KEY]; + expected.sort_unstable(); + assert_eq!(listed, expected, "{listing:#}"); + for (key, family) in [(FIRST_KEY, FIRST_FAMILY), (SECOND_KEY, SECOND_FAMILY)] { + let runtime = listed_runtime(&listing, key); + assert_eq!( + runtime["runtime_id"], + format!("therock-release:{family}"), + "{key}" + ); + assert_eq!(runtime["family"], family, "{key}"); + assert_eq!(runtime["format"], "tarball", "{key}"); + // As recorded, not the text's display form with the build date. + assert_eq!(runtime["version"], PLANTED_VERSION, "{key}"); + assert_eq!(runtime["mode"], "read-only", "{key}"); + assert_eq!(runtime["status"], "ready", "{key}"); + assert_null_field(runtime, "unusable_reason"); + // The planted manifest has no `devel` field, which the CLI reads as + // a legacy install that included the toolchain. + assert_eq!(runtime["toolchain"], "included", "{key}"); + let install_root = runtime["install_root"] + .as_str() + .unwrap_or_else(|| panic!("{key} has no install_root:\n{listing:#}")); + assert_eq!( + Path::new(install_root), + planted_install_root(world, family), + "{key}" + ); + } +} + +#[then("the JSON marks the second runtime active and the first as the rollback target")] +async fn json_marks_active_and_rollback(world: &mut E2eWorld) { + let listing = listing_json(world); + assert_eq!( + listed_keys_with(&listing, "active"), + [SECOND_KEY], + "{listing:#}" + ); + assert_eq!( + listed_keys_with(&listing, "rollback_target"), + [FIRST_KEY], + "{listing:#}" + ); + assert_eq!(listing["active_runtime_key"], SECOND_KEY, "{listing:#}"); + assert_eq!(listing["previous_runtime_key"], FIRST_KEY, "{listing:#}"); + // A healthy state reports no problem, in either form. + assert_null_field(&listing, "active_problem"); + let text = crate::run_rocm_ok(world, &["runtimes", "list"]); + assert!( + !text.contains("active_status:"), + "a healthy listing must print no active_status line:\n{text}" + ); +} + +#[then("the JSON lists no runtimes and no active runtime")] +async fn json_lists_nothing(world: &mut E2eWorld) { + let listing = listing_json(world); + assert_eq!(listing["runtimes"], serde_json::json!([]), "{listing:#}"); + for unset in [ + "default_runtime_id", + "active_runtime_key", + "previous_runtime_key", + "active_problem", + ] { + assert_null_field(&listing, unset); + } + // The paths are reported even with nothing registered, and each is the + // isolated data dir's own: swapping the two would fail here. + let runtimes_dir = registry_dir(world) + .parent() + .expect("the registry has a parent") + .to_path_buf(); + for (field, expected) in [ + ("registry", registry_dir(world)), + ("marker", runtimes_dir.join("active.json")), + ] { + let reported = listing + .get(field) + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| panic!("`{field}` is not a path:\n{listing:#}")); + assert_eq!(Path::new(reported), expected, "`{field}`:\n{listing:#}"); + } +} + +/// Lists the same state as text and compares its `*`/`-` rows with the JSON +/// flags just read, so the two forms are shown to agree, not assumed to. +#[then("the text listing marks the same runtimes as the JSON")] +async fn text_marks_match_json(world: &mut E2eWorld) { + let listing = listing_json(world); + let text = crate::run_rocm_ok(world, &["runtimes", "list"]); + let active = listed_keys_with(&listing, "active"); + // Two empty lists agree too; this step is only evidence where something is + // active. + assert!( + !active.is_empty(), + "the JSON marks no runtime active, so there is nothing to compare:\n{listing:#}" + ); + // The text has one marker column and gives it to `*` when a runtime is + // both active and the rollback target, so `-` shows only the others. + let rollback_only: Vec = listed_keys_with(&listing, "rollback_target") + .into_iter() + .filter(|key| !active.contains(key)) + .collect(); + assert_eq!(text_rows_marked(&text, "*"), active, "{text}"); + assert_eq!(text_rows_marked(&text, "-"), rollback_only, "{text}"); +} + +#[then("the JSON shows the default runtime as typed and no active runtime key")] +async fn json_shows_default_as_typed(world: &mut E2eWorld) { + let listing = listing_json(world); + assert_eq!( + listing["default_runtime_id"], + first_runtime_id_in_other_case(), + "{listing:#}" + ); + assert_null_field(&listing, "active_runtime_key"); +} + +#[then("the JSON marks only the first runtime active and reports no problem")] +async fn json_marks_first_active_without_problem(world: &mut E2eWorld) { + let listing = listing_json(world); + assert_eq!( + listed_keys_with(&listing, "active"), + [FIRST_KEY], + "{listing:#}" + ); + assert_null_field(&listing, "active_problem"); +} + +#[then("the JSON marks only the second runtime active")] +async fn json_marks_only_second_active(world: &mut E2eWorld) { + let listing = listing_json(world); + assert_eq!( + listed_keys_with(&listing, "active"), + [SECOND_KEY], + "{listing:#}" + ); +} + +/// The install root `rocm examine --json` reports (`summary.active_runtime_root`) +/// for the active runtime, which comes from the resolver the commands use. +fn examined_active_runtime_root(world: &E2eWorld) -> PathBuf { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["examine", "--framework", "skip", "--json"]); + assert_eq!(rc, 0, "examine failed:\n{stdout}\n{stderr}"); + let examined: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|error| panic!("examine --json is not JSON ({error}):\n{stdout}")); + let root = examined["summary"]["active_runtime_root"] + .as_str() + .unwrap_or_else(|| panic!("examine reports no active runtime root:\n{examined:#}")); + PathBuf::from(root) +} + +#[then("examine reports the first runtime as the active one")] +async fn examine_reports_first_active(world: &mut E2eWorld) { + assert_eq!( + examined_active_runtime_root(world), + planted_install_root(world, FIRST_FAMILY) + ); +} + +#[then("examine reports the second runtime as the active one")] +async fn examine_reports_second_active(world: &mut E2eWorld) { + assert_eq!( + examined_active_runtime_root(world), + planted_install_root(world, SECOND_FAMILY) + ); +} + +#[then("the text listing says the active runtime key has no registered runtime")] +async fn text_reports_missing_active_key(world: &mut E2eWorld) { + let key = config_active_runtime_key(world).expect("the config names an active runtime key"); + let text = crate::run_rocm_ok(world, &["runtimes", "list"]); + let status: Vec<&str> = text + .lines() + .map(str::trim_start) + .filter(|line| line.starts_with("active_status:")) + .collect(); + assert_eq!( + status, + [format!( + "active_status: missing manifest for active_runtime_key={key}" + )], + "{text}" + ); +} + +#[then("the text listing marks the first runtime active with no active_status line")] +async fn text_marks_first_active_without_status(world: &mut E2eWorld) { + let text = crate::run_rocm_ok(world, &["runtimes", "list"]); + assert_eq!(text_rows_marked(&text, "*"), [FIRST_KEY], "{text}"); + assert!( + !text.contains("active_status:"), + "the runtime it marks active must not also be reported missing:\n{text}" + ); +} + +#[then("the JSON reports the first runtime unusable because its folder is missing")] +async fn json_reports_first_unusable(world: &mut E2eWorld) { + let listing = listing_json(world); + let runtime = listed_runtime(&listing, FIRST_KEY); + assert_eq!(runtime["status"], "unusable", "{listing:#}"); + // The reason is prose for a person, not a code; it is the message + // activation refuses with, naming the folder that is gone. + let folder = planted_install_root(world, FIRST_FAMILY); + assert_eq!( + runtime["unusable_reason"], + format!("install root is missing: {}", folder.display()), + "{listing:#}" + ); +} + +#[then("the JSON reports the second runtime ready")] +async fn json_reports_second_ready(world: &mut E2eWorld) { + let listing = listing_json(world); + let runtime = listed_runtime(&listing, SECOND_KEY); + assert_eq!(runtime["status"], "ready", "{listing:#}"); + assert_null_field(runtime, "unusable_reason"); +} + +/// Names the key the config holds, read back from the config file, and checks +/// that no listed runtime has it in any letter case. +#[then("the JSON reports that the active runtime key has no registered runtime")] +async fn json_reports_missing_active_manifest(world: &mut E2eWorld) { + let listing = listing_json(world); + let key = config_active_runtime_key(world).expect("the config names an active runtime key"); + assert_eq!( + listing["active_problem"], + serde_json::json!({"kind": "missing_manifest_for_key", "runtime_key": key}), + "{listing:#}" + ); + assert_eq!(listing["active_runtime_key"], key.as_str(), "{listing:#}"); + let listed: Vec<&str> = listing["runtimes"] + .as_array() + .expect("`runtimes` is an array") + .iter() + .filter_map(|runtime| runtime["runtime_key"].as_str()) + .collect(); + assert!( + !listed + .iter() + .any(|listed| listed.eq_ignore_ascii_case(&key)), + "`{key}` is registered after all:\n{listing:#}" + ); +} + +/// The failure half of the JSON rules in docs/json-output.md: exit 1 and an +/// empty stdout, so a script that reads stdout cannot mistake the failure for a +/// listing. +#[then("the list fails with nothing printed on stdout")] +async fn list_fails_with_empty_stdout(world: &mut E2eWorld) { + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert_eq!(world.cli_rc, Some(1), "stderr:\n{stderr}"); + assert_eq!( + world.cli_output.as_deref(), + Some(""), + "stdout must be empty when the list fails; stderr:\n{stderr}" + ); +} + +/// The error is the plain text a person reads, not a JSON error document, and +/// it names the entry that could not be read so the user knows what to repair. +#[then("the error on stderr is plain text naming the unreadable registry entry")] +async fn stderr_names_unreadable_entry(world: &mut E2eWorld) { + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert!( + serde_json::from_str::(stderr.trim()).is_err(), + "the error must not be a JSON document:\n{stderr}" + ); + let entry = registry_dir(world).join(format!("{FIRST_KEY}.json")); + assert!( + stderr.contains(&format!("failed to read {}", entry.display())), + "stderr must name {}:\n{stderr}", + entry.display() + ); +} + +#[then("the JSON marks no runtime active")] +async fn json_marks_none_active(world: &mut E2eWorld) { + let listing = listing_json(world); + assert!( + listed_keys_with(&listing, "active").is_empty(), + "{listing:#}" + ); +} + #[then("the CLI refuses and requires --yes")] async fn uninstall_refused_without_yes(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no command rc recorded"); @@ -1402,3 +2066,166 @@ async fn server_record_still_names_first_runtime(world: &mut E2eWorld) { the live one it was:\n{record}" ); } + +#[then( + "the nightly runtime's listing names its channel, package source, source layout and exact GPU target" +)] +async fn nightly_listing_names_provenance(world: &mut E2eWorld) { + let out = ok_output(world); + assert_eq!( + listed_provenance(&out, NIGHTLY_EXACT_TARGET_KEY), + format!( + "channel=nightly source={NIGHTLY_SOURCE} \ + source_layout_generation={NIGHTLY_SOURCE_LAYOUT} \ + device_target={NIGHTLY_DEVICE_TARGET}" + ), + "the listing must name where the nightly runtime came from:\n{out}" + ); +} + +#[then("its version shows its build date")] +async fn nightly_version_shows_build_date(world: &mut E2eWorld) { + let out = ok_output(world); + let row = out + .lines() + .find(|line| line.contains(&format!(" {NIGHTLY_EXACT_TARGET_KEY} runtime_id="))) + .unwrap_or_else(|| panic!("no row listed for the nightly runtime:\n{out}")); + assert!( + row.contains(" version=10.1.0a20260822 (build 2026-08-22) "), + "the nightly runtime's version must show its build date:\n{row}" + ); +} + +#[then( + "the older runtime's listing says which of those facts were not recorded instead of guessing" +)] +async fn older_listing_says_unrecorded(world: &mut E2eWorld) { + let out = ok_output(world); + assert_eq!( + listed_provenance(&out, FIRST_KEY), + format!( + "channel=release source=https://example.invalid/{FIRST_KEY}.tar.gz \ + source_layout_generation=unrecorded device_target=unrecorded" + ), + "an unrecorded fact must say so, not be inferred from the family:\n{out}" + ); +} + +#[then("the adopted runtime's listing says it has no recorded package source")] +async fn adopted_listing_says_source_unrecorded(world: &mut E2eWorld) { + let out = ok_output(world); + assert_eq!( + listed_provenance(&out, ADOPTED_KEY), + "channel=release source=unrecorded \ + source_layout_generation=unrecorded device_target=unrecorded", + "adoption installed nothing, so the listing must not present the \ + adoption marker as a package source:\n{out}" + ); +} + +/// The `provenance` object the JSON listing gives `runtime_key`, compared whole +/// against `expected` by the steps below, so a fact under another name, a +/// missing one, or the text's `unrecorded` in place of `null` fails. +fn assert_json_provenance(world: &E2eWorld, runtime_key: &str, expected: &serde_json::Value) { + let listing = listing_json(world); + assert_eq!( + &listed_runtime(&listing, runtime_key)["provenance"], + expected, + "{runtime_key}:\n{listing:#}" + ); +} + +#[then( + "the JSON names the nightly runtime's channel, package source, source layout and exact GPU target" +)] +async fn json_names_nightly_provenance(world: &mut E2eWorld) { + assert_json_provenance( + world, + NIGHTLY_EXACT_TARGET_KEY, + &serde_json::json!({ + "channel": "nightly", + "source": NIGHTLY_SOURCE, + "source_layout_generation": NIGHTLY_SOURCE_LAYOUT, + "device_target": NIGHTLY_DEVICE_TARGET, + }), + ); +} + +/// The older runtime's family is the exact `gfx942`, so a target guessed from +/// it would be a plausible string here rather than `null`. +#[then( + "the JSON gives no value for each fact the older runtime did not record instead of guessing" +)] +async fn json_gives_null_for_unrecorded_facts(world: &mut E2eWorld) { + assert_json_provenance( + world, + FIRST_KEY, + &serde_json::json!({ + "channel": "release", + "source": format!("https://example.invalid/{FIRST_KEY}.tar.gz"), + "source_layout_generation": null, + "device_target": null, + }), + ); +} + +#[then("the JSON gives no package source for the adopted runtime")] +async fn json_gives_adopted_runtime_no_source(world: &mut E2eWorld) { + assert_json_provenance( + world, + ADOPTED_KEY, + &serde_json::json!({ + "channel": "release", + "source": null, + "source_layout_generation": null, + "device_target": null, + }), + ); +} + +/// Lists the same state as text and compares every runtime's `provenance:` line +/// with the JSON object just read: the same names, the same values, and +/// `unrecorded` exactly where the JSON has `null`. +#[then("the text listing names the same provenance as the JSON for every runtime")] +async fn text_provenance_matches_json(world: &mut E2eWorld) { + let listing = listing_json(world); + // The listing run consumed the scenario's setting; this run sets it again. + disable_startup_update_check(world); + let (text, stderr, rc) = crate::run_rocm_with_scenario_env(world, &["runtimes", "list"]); + assert_eq!(rc, 0, "the text listing failed:\n{text}\n{stderr}"); + let runtimes = listing["runtimes"] + .as_array() + .unwrap_or_else(|| panic!("`runtimes` is not an array:\n{listing:#}")); + assert!( + !runtimes.is_empty(), + "the JSON lists no runtime, so there is nothing to compare:\n{listing:#}" + ); + for runtime in runtimes { + let key = runtime["runtime_key"] + .as_str() + .unwrap_or_else(|| panic!("a runtime has no runtime_key:\n{listing:#}")); + let object = runtime["provenance"] + .as_object() + .unwrap_or_else(|| panic!("{key} has no provenance object:\n{listing:#}")); + let shown: Vec<(&str, &str)> = listed_provenance(&text, key) + .split(' ') + .map(|pair| { + pair.split_once('=') + .unwrap_or_else(|| panic!("{key}: {pair:?} is not name=value:\n{text}")) + }) + .collect(); + assert_eq!( + shown.len(), + object.len(), + "{key}: the text and the JSON name different facts:\n{text}\n{listing:#}" + ); + for (name, value) in shown { + let expected = match object.get(name) { + Some(serde_json::Value::Null) => "unrecorded", + Some(serde_json::Value::String(recorded)) => recorded.as_str(), + other => panic!("{key}: the JSON's {name} is {other:?}:\n{listing:#}"), + }; + assert_eq!(value, expected, "{key}: {name}:\n{text}\n{listing:#}"); + } + } +} diff --git a/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs new file mode 100644 index 0000000000..00ab793e39 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs @@ -0,0 +1,800 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for the side-by-side scenarios in `runtime_lifecycle.feature`: three +//! managed wheel runtimes for one GPU family — a ROCm 7 runtime installed before +//! the ROCm 10 layout existed, a ROCm 10 release runtime from that layout, and a +//! ROCm 10 nightly prerelease — listed, activated, uninstalled and selected by +//! the name two of them share. +//! +//! The runtimes are planted, not installed: a real install downloads several +//! GiB of wheels. Each record holds what the wheel install in `therock.rs` +//! writes for its role (see [`Role`]): a registry entry and the same manifest in +//! the folder's own `.rocm-cli-runtime.json`, with every field the writing +//! version serialized. The readiness check (`runtime_usability_status` in +//! `apps/rocm/src/runtime_registry.rs`) reads none of the files it checks and +//! runs nothing: it asks for the install folder and the probe's SDK root and bin +//! to be folders, and for the folder's local manifest, the Python executable and +//! a listed `amdhip64` and `hipblas` library to be files, so stub files stand in +//! for all of them. +//! +//! The local manifest is a full copy rather than the `{}` some other fixtures +//! write, because uninstall parses it before deciding the folder's fate +//! (`local_runtime_manifest_matches` in `apps/rocm/src/runtime_uninstall.rs`): +//! one that does not parse as a manifest fails the uninstall with a parse +//! error, and only one naming the same key, id and folder as the registry entry +//! lets the folder be removed. +//! +//! Every command runs with the startup update check disabled. With a runtime +//! installed, a command otherwise first asks the package index whether that +//! runtime has an update, which is network traffic and cache writes these +//! scenarios are not about. + +use std::path::{Path, PathBuf}; + +use cucumber::{given, then, when}; +use e2e_cucumber::cli_failure_report; +use e2e_cucumber::fixture_binary::fixture_target_name; +use sha2::{Digest as _, Sha256}; + +use super::comfyui_steps::disable_startup_update_check; +use super::runtime_lifecycle_steps::{ + active_marker_path, config_path, read_json_if_present, registry_dir, +}; +use crate::E2eWorld; + +/// Shared by the ROCm 7 and the ROCm 10 release runtimes: `rocm install sdk` +/// derives the id from the channel and GPU family alone, so both carry it, and +/// only their keys tell them apart. +const SHARED_RELEASE_RUNTIME_ID: &str = "therock-release:gfx94X-dcgpu"; +const NIGHTLY_RUNTIME_ID: &str = "therock-nightly:gfx94X-dcgpu"; +/// The family `rocm install sdk` resolves for a host reporting `gfx942`. +const FAMILY: &str = "gfx94X-dcgpu"; +const DEVICE_TARGET: &str = "gfx942"; + +/// One of the three planted runtimes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + /// ROCm 7.14.0 from the canonical release stream, installed by the CLI as + /// it stood just before the ROCm 10 layout was added (commit `a6fa76db`'s + /// parent). That install already recorded a wheel composition and keyed the + /// runtime by its fingerprint, but wrote neither `source_layout_generation` + /// nor `devel`, which did not exist yet, and always installed the compiler + /// toolchain. + Rocm7, + /// ROCm 10.0.0, a stable release and so on the release channel only, from + /// the ROCm 10 (`next`) layout, installed by the current CLI without the + /// toolchain (now opt-in). + Rocm10Release, + /// The ROCm 10.1 nightly prerelease `10.1.0a20260822`, the last nightly the + /// canonical layout published: nightlies from 2026-08-23 on come from the + /// `next` layout instead. So this one records the canonical layout + /// (`multi-arch-v2`), installed by the current CLI without the toolchain. + Rocm10Nightly, +} + +impl Role { + const ALL: [Self; 3] = [Self::Rocm7, Self::Rocm10Release, Self::Rocm10Nightly]; + + const fn channel(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Release => "release", + Self::Rocm10Nightly => "nightly", + } + } + + const fn runtime_id(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Release => SHARED_RELEASE_RUNTIME_ID, + Self::Rocm10Nightly => NIGHTLY_RUNTIME_ID, + } + } + + const fn version(self) -> &'static str { + match self { + Self::Rocm7 => "7.14.0", + Self::Rocm10Release => "10.0.0", + Self::Rocm10Nightly => "10.1.0a20260822", + } + } + + /// The layout generation the install recorded in its composition. + const fn layout_generation(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Nightly => "multi-arch-v2", + Self::Rocm10Release => "next-v1", + } + } + + /// The pip index the install read from, which it records both as + /// `index_url` and as `selected_artifact_url`. The hosts stand in for the + /// published ones the way `ROCM_CLI_THEROCK_*_PIP_BASE` overrides do; the + /// install records whichever base it resolved. + const fn index_url(self) -> &'static str { + match self { + Self::Rocm7 => "https://example.invalid/rocm/whl-multi-arch", + Self::Rocm10Release => "https://example.invalid/rocm/whl-next", + Self::Rocm10Nightly => "https://example.invalid/nightlies/whl-multi-arch", + } + } + + /// Whether the install included the compiler toolchain: always before it + /// became opt-in, and not by default since. + const fn includes_devel(self) -> bool { + matches!(self, Self::Rocm7) + } + + /// `torch`, `torchvision` and `torchaudio` as the install pinned them. + const fn torch_stack(self) -> [&'static str; 3] { + match self { + Self::Rocm7 => [ + "2.11.0+rocm7.14.0", + "0.26.0+rocm7.14.0", + "2.11.0+rocm7.14.0", + ], + Self::Rocm10Release => [ + "2.13.0+rocm10.0.0", + "0.28.0+rocm10.0.0", + "2.11.0.2+rocm10.0.0", + ], + Self::Rocm10Nightly => [ + "2.12.0+rocm10.1.0a20260822", + "0.27.0+rocm10.1.0a20260822", + "2.12.0+rocm10.1.0a20260822", + ], + } + } + + /// The exact specs handed to `uv`, as `therock_pip_package_specs` builds + /// them. + fn package_specs(self) -> Vec { + let device = format!("device-{DEVICE_TARGET}"); + let extras = if self.includes_devel() { + "libraries,devel" + } else { + "libraries" + }; + let [torch, torchvision, torchaudio] = self.torch_stack(); + vec![ + format!("rocm[{extras},{device}]=={}", self.version()), + format!("torch[{device}]=={torch}"), + format!("torchvision[{device}]=={torchvision}"), + format!("torchaudio=={torchaudio}"), + ] + } + + /// The key `wheel_runtime_key` in `apps/rocm/src/therock.rs` derives: the + /// channel and version, then the first 16 hex digits of a SHA-256 over the + /// composition. The derivation is the same before and after the ROCm 10 + /// layout, so the layout generation inside the composition is what keeps a + /// ROCm 10 key from ever colliding with a canonical one. Planting checks the + /// result against [`Self::expected_key`], the literal keys the unit test + /// `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in `therock.rs` + /// derives with the production code from the same inputs. + fn key(self) -> String { + let mut hasher = Sha256::new(); + hasher.update(self.layout_generation().as_bytes()); + for spec in self.package_specs() { + hasher.update([0]); + hasher.update(spec.as_bytes()); + } + hasher.update([0]); + hasher.update(b"rocm-sdk-target"); + hasher.update([0]); + hasher.update(DEVICE_TARGET.as_bytes()); + let fingerprint = format!("{:x}", hasher.finalize()); + format!( + "{}-wheel-multi-arch-{}-{}", + self.channel(), + self.version(), + &fingerprint[..16] + ) + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '-' + } + }) + .collect() + } + + /// The key [`Self::key`] must produce: the literals + /// `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in `therock.rs` + /// gets from the production `wheel_runtime_key` for the same inputs. The + /// unit test ties these literals to the production derivation, and + /// planting ties them to this fixture, so neither side can drift alone. + const fn expected_key(self) -> &'static str { + match self { + Self::Rocm7 => "release-wheel-multi-arch-7-14-0-992a08553d8be8f0", + Self::Rocm10Release => "release-wheel-multi-arch-10-0-0-3d0e256f664a960d", + Self::Rocm10Nightly => "nightly-wheel-multi-arch-10-1-0a20260822-95e3a823499c530b", + } + } + + /// What `rocm runtimes list` must report for the compiler toolchain. + const fn toolchain(self) -> &'static str { + if self.includes_devel() { + "included" + } else { + "excluded" + } + } +} + +/// A planted runtime and the bytes of its two install records as planted. +#[derive(Debug)] +struct PlantedRuntime { + role: Role, + key: String, + install_root: PathBuf, + registry_path: PathBuf, + registry_bytes: Vec, + local_manifest_bytes: Vec, +} + +impl PlantedRuntime { + fn local_manifest_path(&self) -> PathBuf { + self.install_root.join(".rocm-cli-runtime.json") + } +} + +/// What one `rocm runtimes activate` left behind. +#[derive(Debug)] +struct Activation { + role: Role, + in_use: InUse, +} + +/// What the CLI's two records of the runtime in use name: the config +/// (`config.json`) and the active-runtime marker (`/runtimes/active.json`). +/// `None` throughout means the record names nothing, or does not exist. +#[derive(Debug, PartialEq, Eq)] +struct InUse { + config_active: Option, + config_previous: Option, + config_default_id: Option, + marker_key: Option, + marker_id: Option, + marker_previous_key: Option, + marker_previous_id: Option, +} + +/// Per-scenario state, held on the World. +#[derive(Debug)] +pub struct SideBySide { + runtimes: Vec, + activations: Vec, +} + +impl SideBySide { + fn runtime(&self, role: Role) -> &PlantedRuntime { + self.runtimes + .iter() + .find(|runtime| runtime.role == role) + .unwrap_or_else(|| panic!("{role:?} runtime was not planted")) + } + + fn key(&self, role: Role) -> &str { + &self.runtime(role).key + } +} + +const fn state(world: &E2eWorld) -> &SideBySide { + world + .side_by_side + .as_ref() + .expect("no side-by-side runtimes were planted in this scenario") +} + +fn key(world: &E2eWorld, role: Role) -> String { + state(world).key(role).to_owned() +} + +fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", parent.display())); + } + std::fs::write(path, contents) + .unwrap_or_else(|error| panic!("failed to write {}: {error}", path.display())); +} + +/// The `site-packages` of a venv at `install_root`, as Python reports it. +fn site_packages(install_root: &Path) -> PathBuf { + if cfg!(windows) { + install_root.join("Lib").join("site-packages") + } else { + install_root + .join("lib") + .join("python3.12") + .join("site-packages") + } +} + +/// Where `rocm_sdk.find_libraries` finds `shortname` inside a package root. +fn library_path(package_root: &Path, shortname: &str) -> PathBuf { + if cfg!(windows) { + package_root.join("bin").join(format!("{shortname}.dll")) + } else { + package_root.join("lib").join(format!("lib{shortname}.so")) + } +} + +/// The `rocm_sdk` probe `rocm install sdk` records, in the shape the probe +/// script in `apps/rocm/src/therock/probe.rs` prints: every package root's `bin` +/// and `lib` folders, the venv's scripts folder, and an SDK root that is the +/// `devel` package when the toolchain is installed and the first package root +/// otherwise, with a CMake path only for `devel`. The script also lists the +/// installed `rocm*` distributions under `packages`; that list is left empty +/// here because nothing on these paths (listing, activating, uninstalling) +/// reads it. Two libraries stand for the full list, being the two the +/// readiness check asks for. Creates the files and folders the record names. +fn plant_rocm_sdk_probe(role: Role, install_root: &Path, scripts: &Path) -> serde_json::Value { + let site_packages = site_packages(install_root); + let core = site_packages.join("_rocm_sdk_core"); + let libraries = site_packages.join(format!("_rocm_sdk_libraries_{FAMILY}")); + let devel = site_packages.join("_rocm_sdk_devel"); + let mut runtime_roots = vec![core.clone(), libraries.clone()]; + if role.includes_devel() { + runtime_roots.insert(0, devel); + } + let root = runtime_roots[0].clone(); + let mut bin_paths: Vec = runtime_roots.iter().map(|root| root.join("bin")).collect(); + bin_paths.push(scripts.to_path_buf()); + let library_paths: Vec = runtime_roots + .iter() + .flat_map(|root| [root.join("bin"), root.join("lib")]) + .collect(); + for dir in &library_paths { + std::fs::create_dir_all(dir) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", dir.display())); + } + let amdhip = library_path(&core, "amdhip64"); + let hipblas = library_path(&libraries, "hipblas"); + for file in [&amdhip, &hipblas] { + write_file(file, b"stub"); + } + serde_json::json!({ + "import_ok": true, + "rocm_sdk_version": role.version(), + "site_packages": site_packages, + "root_path": root, + "bin_path": root.join("bin"), + "cmake_path": role.includes_devel().then(|| root.join("lib").join("cmake")), + "runtime_roots": runtime_roots, + "bin_paths": bin_paths, + "library_paths": library_paths, + "default_target_family": DEVICE_TARGET, + "available_target_families": [DEVICE_TARGET], + "resolved_target_family": DEVICE_TARGET, + "packages": [], + "library_shortnames": ["amdhip64", "hipblas"], + "resolved_libraries": [ + {"shortname": "amdhip64", "paths": [amdhip]}, + {"shortname": "hipblas", "paths": [hipblas]}, + ], + "error": null, + }) +} + +/// The manifest the wheel install writes for `role`, with every field the +/// writing version serialized, and the files its readiness check asks for. +fn plant_manifest(world: &E2eWorld, role: Role, key: &str) -> (PathBuf, serde_json::Value) { + let data_dir = world.app_paths().data_dir; + let install_root = data_dir.join("runtimes").join("wheel").join(key); + let scripts = install_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python = scripts.join(fixture_target_name("python")); + write_file(&python, b"stub"); + // The interpreter the venv was created from: ROCm CLI's managed Python. + let python_launcher = data_dir + .join("tools") + .join("python") + .join(fixture_target_name("python")); + let [torch, ..] = role.torch_stack(); + + let mut manifest = serde_json::json!({ + "runtime_key": key, + "runtime_id": role.runtime_id(), + "channel": role.channel(), + "format": "wheel", + "family": FAMILY, + "family_source": "host", + "version": role.version(), + "install_root": install_root, + "selected_artifact_url": role.index_url(), + "index_url": role.index_url(), + "tarball_file_name": null, + "python_launcher": python_launcher, + "python_executable": python, + "pip_cache_dir": null, + "rocm_sdk": plant_rocm_sdk_probe(role, &install_root, &scripts), + "sdk_torch": torch, + "wheel_composition": { + "source_layout_generation": role.layout_generation(), + "package_specs": role.package_specs(), + "rocm_sdk_target": DEVICE_TARGET, + }, + "read_only": false, + "imported_from": null, + "installed_at_unix_ms": match role { + Role::Rocm7 => 1_757_000_000_000u64, + Role::Rocm10Release => 1_762_000_000_000, + Role::Rocm10Nightly => 1_762_500_000_000, + }, + }); + if role != Role::Rocm7 { + let fields = manifest + .as_object_mut() + .expect("the manifest is a JSON object"); + fields.insert( + "source_layout_generation".into(), + role.layout_generation().into(), + ); + fields.insert("devel".into(), role.includes_devel().into()); + } + (install_root, manifest) +} + +/// Runs `rocm` with the startup update check disabled. `command_env` is +/// consumed by each run, so it is set again for every one. +fn run(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { + disable_startup_update_check(world); + crate::run_rocm_with_scenario_env(world, args) +} + +/// [`run`], recording the outcome on the World for the Then steps. +fn run_recorded(world: &mut E2eWorld, args: &[&str]) { + let (stdout, stderr, rc) = run(world, args); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// [`run`], failing the step with the full diagnostic bundle unless it exits 0. +/// For actions that only set the stage or have their outcome checked on disk. +fn run_ok(world: &mut E2eWorld, args: &[&str]) { + let (stdout, stderr, rc) = run(world, args); + assert!( + rc == 0, + "{}", + cli_failure_report(args, rc, &stdout, &stderr) + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn string_field(value: Option<&serde_json::Value>, field: &str) -> Option { + value + .and_then(|value| value.get(field)) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) +} + +/// What the config and the active-runtime marker name now. +fn in_use(world: &E2eWorld) -> InUse { + let config = read_json_if_present(&config_path(world)); + let marker = read_json_if_present(&active_marker_path(world)); + InUse { + config_active: string_field(config.as_ref(), "active_runtime_key"), + config_previous: string_field(config.as_ref(), "previous_runtime_key"), + config_default_id: string_field(config.as_ref(), "default_runtime_id"), + marker_key: string_field(marker.as_ref(), "runtime_key"), + marker_id: string_field(marker.as_ref(), "runtime_id"), + marker_previous_key: string_field(marker.as_ref(), "previous_runtime_key"), + marker_previous_id: string_field(marker.as_ref(), "previous_runtime_id"), + } +} + +/// What both records must name while `role` is in use with `previous` as its +/// rollback target. +fn expected_in_use(world: &E2eWorld, role: Role, previous: Option) -> InUse { + let role_key = Some(key(world, role)); + let previous_key = previous.map(|previous| key(world, previous)); + let previous_id = previous.map(|previous| previous.runtime_id().to_owned()); + InUse { + config_active: role_key.clone(), + config_previous: previous_key.clone(), + config_default_id: Some(role.runtime_id().to_owned()), + marker_key: role_key, + marker_id: Some(role.runtime_id().to_owned()), + marker_previous_key: previous_key, + marker_previous_id: previous_id, + } +} + +fn activate(world: &mut E2eWorld, role: Role) { + let key = key(world, role); + run_ok(world, &["runtimes", "activate", &key]); + let in_use = in_use(world); + world + .side_by_side + .as_mut() + .expect("no side-by-side runtimes were planted in this scenario") + .activations + .push(Activation { role, in_use }); +} + +fn assert_in_use(world: &E2eWorld, role: Role, previous: Option) { + assert_eq!( + in_use(world), + expected_in_use(world, role, previous), + "the config and the active-runtime marker must name the {role:?} runtime as in use, \ + with {previous:?} as its rollback target" + ); +} + +/// Fails unless `runtime`'s folder is still there and both its install records +/// hold exactly the bytes that were planted. +fn assert_records_untouched(runtime: &PlantedRuntime) { + let role = runtime.role; + assert!( + runtime.install_root.is_dir(), + "the {role:?} runtime's folder was removed: {}", + runtime.install_root.display() + ); + for (path, planted) in [ + (runtime.registry_path.clone(), &runtime.registry_bytes), + (runtime.local_manifest_path(), &runtime.local_manifest_bytes), + ] { + let now = std::fs::read(&path).unwrap_or_else(|error| { + panic!( + "the {role:?} runtime's record {} is gone: {error}", + path.display() + ) + }); + assert!( + &now == planted, + "the {role:?} runtime's record {} was rewritten.\nplanted:\n{}\nnow:\n{}", + path.display(), + String::from_utf8_lossy(planted), + String::from_utf8_lossy(&now) + ); + } +} + +fn output_text(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given( + "a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family" +)] +async fn plant_side_by_side_runtimes(world: &mut E2eWorld) { + let registry = registry_dir(world); + let runtimes = Role::ALL + .into_iter() + .map(|role| { + let key = role.key(); + assert_eq!( + key, + role.expected_key(), + "the {role:?} fixture no longer derives the key the production code \ + gives these inputs; update `Role::expected_key` together with \ + `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in \ + apps/rocm/src/therock.rs" + ); + let (install_root, manifest) = plant_manifest(world, role, &key); + let manifest = serde_json::to_vec_pretty(&manifest) + .expect("failed to serialize the runtime manifest"); + let registry_path = registry.join(format!("{key}.json")); + write_file(&install_root.join(".rocm-cli-runtime.json"), &manifest); + write_file(®istry_path, &manifest); + PlantedRuntime { + role, + key, + install_root, + registry_path, + registry_bytes: manifest.clone(), + local_manifest_bytes: manifest, + } + }) + .collect(); + world.side_by_side = Some(SideBySide { + runtimes, + activations: Vec::new(), + }); +} + +#[given("the nightly runtime is in use after the ROCm 10 release runtime")] +async fn nightly_in_use_after_rocm10_release(world: &mut E2eWorld) { + activate(world, Role::Rocm10Release); + activate(world, Role::Rocm10Nightly); + assert_in_use(world, Role::Rocm10Nightly, Some(Role::Rocm10Release)); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user lists the installed runtimes")] +async fn list_installed_runtimes(world: &mut E2eWorld) { + run_recorded(world, &["runtimes", "list"]); +} + +#[when("the user activates each of the three runtimes in turn")] +async fn activate_each_in_turn(world: &mut E2eWorld) { + for role in Role::ALL { + activate(world, role); + } +} + +#[when("the user uninstalls the ROCm 10 release runtime")] +async fn uninstall_rocm10_release(world: &mut E2eWorld) { + let key = key(world, Role::Rocm10Release); + run_ok(world, &["runtimes", "uninstall", &key, "--yes"]); +} + +#[when("the user uninstalls the runtime in use")] +async fn uninstall_runtime_in_use(world: &mut E2eWorld) { + let in_use = in_use(world) + .config_active + .expect("the config names no runtime in use to uninstall"); + run_ok(world, &["runtimes", "uninstall", &in_use, "--yes"]); +} + +#[when("the user activates the ROCm 7 runtime")] +async fn activate_rocm7(world: &mut E2eWorld) { + activate(world, Role::Rocm7); +} + +#[when("the user activates a runtime by the name the two release runtimes share")] +async fn activate_by_shared_name(world: &mut E2eWorld) { + run_recorded(world, &["runtimes", "activate", SHARED_RELEASE_RUNTIME_ID]); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("all three runtimes are listed as ready, each under its own key")] +async fn all_three_listed_ready(world: &mut E2eWorld) { + let stdout = world.cli_output.clone().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "`rocm runtimes list` failed:\n{}", + output_text(world) + ); + let rows: Vec<&str> = stdout + .lines() + .filter(|line| line.contains(" runtime_id=")) + .collect(); + assert_eq!( + rows.len(), + 3, + "expected one row per planted runtime:\n{stdout}" + ); + for role in Role::ALL { + let prefix = format!("{} runtime_id={} ", key(world, role), role.runtime_id()); + let matching: Vec<&&str> = rows.iter().filter(|row| row.contains(&prefix)).collect(); + let [row] = matching.as_slice() else { + panic!("expected exactly one row for the {role:?} runtime ({prefix}):\n{stdout}"); + }; + assert!( + row.contains(" status=ready "), + "the {role:?} runtime must be listed as ready: {row}" + ); + assert!( + row.ends_with(&format!(" toolchain={}", role.toolchain())), + "the {role:?} runtime must be listed with toolchain={}: {row}", + role.toolchain() + ); + } +} + +#[then("each one becomes the runtime in use in turn")] +async fn each_becomes_in_use(world: &mut E2eWorld) { + let activations = &state(world).activations; + let order: Vec = activations + .iter() + .map(|activation| activation.role) + .collect(); + assert_eq!(order, Role::ALL, "the activations did not all run"); + let mut previous = None; + for activation in activations { + assert_eq!( + activation.in_use, + expected_in_use(world, activation.role, previous), + "activating the {:?} runtime did not make it the one in use, with {previous:?} \ + as its rollback target", + activation.role + ); + previous = Some(activation.role); + } +} + +#[then("none of the three runtimes' install records was rewritten")] +async fn no_record_rewritten(world: &mut E2eWorld) { + for runtime in &state(world).runtimes { + assert_records_untouched(runtime); + } +} + +#[then("that runtime's folder and record are gone")] +async fn rocm10_release_gone(world: &mut E2eWorld) { + let runtime = state(world).runtime(Role::Rocm10Release); + assert!( + !runtime.install_root.exists(), + "the uninstalled runtime's folder is still there: {}\n{}", + runtime.install_root.display(), + output_text(world) + ); + assert!( + !runtime.registry_path.exists(), + "the uninstalled runtime's registry entry is still there: {}", + runtime.registry_path.display() + ); +} + +#[then("the other two runtimes keep their folders and unchanged records")] +async fn others_untouched(world: &mut E2eWorld) { + let state = state(world); + for role in [Role::Rocm7, Role::Rocm10Nightly] { + assert_records_untouched(state.runtime(role)); + } +} + +#[then("the nightly runtime is still in use, with no rollback target left")] +async fn nightly_still_in_use(world: &mut E2eWorld) { + assert_in_use(world, Role::Rocm10Nightly, None); +} + +#[then("no runtime is in use")] +async fn no_runtime_in_use(world: &mut E2eWorld) { + let nothing = InUse { + config_active: None, + config_previous: None, + config_default_id: None, + marker_key: None, + marker_id: None, + marker_previous_key: None, + marker_previous_id: None, + }; + assert_eq!( + in_use(world), + nothing, + "neither the config nor the active-runtime marker may name a runtime in use or a \ + rollback target:\n{}", + output_text(world) + ); +} + +#[then("the ROCm 7 runtime is the runtime in use")] +async fn rocm7_in_use(world: &mut E2eWorld) { + assert_in_use(world, Role::Rocm7, None); +} + +#[then("the CLI refuses and names both release runtimes")] +async fn refused_naming_both_release_runtimes(world: &mut E2eWorld) { + let text = output_text(world); + assert_ne!( + world.cli_rc, + Some(0), + "a name shared by two runtimes must be refused:\n{text}" + ); + for role in [Role::Rocm7, Role::Rocm10Release] { + let key = key(world, role); + assert!( + text.contains(&key), + "the refusal must name the {role:?} runtime ({key}):\n{text}" + ); + } + let nightly = key(world, Role::Rocm10Nightly); + assert!( + !text.contains(&nightly), + "the refusal must not name the nightly runtime, which does not share the name:\n{text}" + ); + // The way out it names works: runtime-lifecycle-19's "each one becomes the + // runtime in use in turn" runs `rocm runtimes activate ` for + // each of these runtimes (`activate`, above) and checks it takes effect. + assert!( + text.contains("activate one by runtime_key"), + "the refusal must say to activate one of them by its key:\n{text}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index d0c92f5f5d..1912039f1e 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -642,22 +642,128 @@ async fn user_reinstalls_lemonade_engine(world: &mut E2eWorld) { world.cli_stderr = Some(stderr); } -/// Verified against real hardware (Strix Halo, gfx1151): a fresh managed SDK -/// install's version does not match Lemonade's packaged pin, Tier 1's install -/// 404s (the pinned build predates a ROCm-7.14 asset), and Tier 2's newest -/// build succeeds -- producing exactly this line. This is the one part of the -/// alignment path with no other e2e coverage: the unit tests exercise the -/// Tier 1/Tier 2/revert state machine directly (against injected install/align -/// steps), but nothing else asserts that `rocm engines install lemonade` -/// actually surfaces the outcome to the user. -#[then("the CLI reports that Lemonade's ROCm backend was aligned to the active SDK")] -async fn assert_lemonade_backend_alignment_reported(world: &mut E2eWorld) { +/// The pin Lemonade installed against ends up equal to the active SDK, by +/// whichever route this host takes: +/// +/// - the active SDK differs from Lemonade's packaged `therock.version`, and the +/// install aligns the pin to it and says so ("Aligned Lemonade's ROCm llama.cpp +/// backend ..."), as on an SDK older than the pin (ROCm 7.14 against +/// Lemonade 2026.44.1's 10.1.0, whose pinned llama.cpp build also ships a 7.14 +/// archive); +/// - the active SDK already is the packaged pin (a ROCm 10.1 SDK), so there is +/// nothing to align: no alignment line, no alignment attempt, and the ROCm +/// backend installs as packaged. +/// +/// Either way the pin in the installed `backend_versions.json` is read back and +/// compared with the active runtime's own version from `rocm runtimes list`, and +/// the selected backend must be `rocm`: a failed alignment reverts the pin to the +/// packaged value and fails the comparison, and a message without the matching +/// pin fails it too. The unit tests drive the Tier 1/Tier 2/revert state machine +/// against injected steps; this is the only check that `rocm engines install +/// lemonade` reaches a ROCm backend paired with the SDK on real hardware. +#[then("Lemonade's ROCm backend is paired with the active SDK")] +async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); assert!( output - .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"), - "expected the install to report the ROCm backend alignment outcome:\n{output}" + .contains("Selected the best supported llama.cpp backend for this GPU: llamacpp:rocm"), + "expected Lemonade's ROCm llama.cpp backend to be selected:\n{output}" ); + let pinned_version = installed_therock_pin(output); + let active_version = active_runtime_version(world); + if !is_plain_release_version(&active_version) { + // A dated (nightly) SDK is left alone by design: no alignment is + // attempted and the packaged pin stays. + assert_eq!( + pinned_version, + packaged_therock_pin(output), + "a nightly SDK ({active_version}) must leave Lemonade's packaged pin in place" + ); + assert!( + !output.contains("Aligned Lemonade's ROCm") + && !stderr.contains("Aligned Lemonade's ROCm"), + "a nightly SDK ({active_version}) was aligned to:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + return; + } + assert_eq!( + pinned_version, active_version, + "the installed backend_versions.json does not pin the active ROCm SDK:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + let packaged_version = packaged_therock_pin(output); + let reported_aligned = output + .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); + if pinned_version == packaged_version { + // The active SDK is the packaged pin (ROCm 10.1 against Lemonade + // 2026.44.1), so the comparison above holds whatever the CLI did. The + // llama.cpp tag paired with that pin is a second signal: an alignment + // that rewrote the pin would also have moved it. + assert_eq!( + installed_backend_pin(output, "llamacpp", "rocm-stable"), + packaged_backend_pin(output, "llamacpp", "rocm-stable"), + "the active SDK is the packaged pin, yet the llama.cpp ROCm tag was rewritten:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + // No alignment is attempted at all then, so none may be reported or + // logged. + assert!( + !reported_aligned + && !stderr.contains("Aligned Lemonade's ROCm backend to") + && !stderr.contains("could not align Lemonade's ROCm backend to"), + "the active SDK is the packaged pin, yet an alignment ran:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + } else { + // The pin moved from the packaged {packaged_version} to the active SDK, + // so an alignment ran, and the install must say so: a CLI that aligns + // silently fails here. + assert!( + reported_aligned, + "the pin moved from the packaged {packaged_version} to {pinned_version}, but the \ + install did not report the alignment:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + } +} + +/// Whether `version` is a plain `major.minor.patch` release, the only kind +/// Lemonade's backend alignment matches against (a nightly such as +/// `7.14.0a20260601` is left alone). +fn is_plain_release_version(version: &str) -> bool { + let parts: Vec<&str> = version.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) +} + +/// `therock.version` in the `resources/backend_versions.json` of the runtime +/// tree the install reports (`env_path:`). +fn installed_therock_pin(output: &str) -> String { + installed_backend_pin(output, "therock", "version") + .unwrap_or_else(|| panic!("no therock.version in the installed backend_versions.json")) +} + +/// `
.` in the installed `resources/backend_versions.json`, e.g. +/// `llamacpp.rocm-stable`, the llama.cpp tag that goes with the ROCm pin. +fn installed_backend_pin(output: &str, section: &str, key: &str) -> Option { + let env_path = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_path: ")) + .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); + let backend_versions_path = + std::path::Path::new(env_path).join("resources/backend_versions.json"); + let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { + panic!( + "failed to read {}: {error}", + backend_versions_path.display() + ) + }); + let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { + panic!( + "failed to parse {}: {error}", + backend_versions_path.display() + ) + }); + parsed[section][key].as_str().map(str::to_owned) } /// The Lemonade install `rocm serve` consults before it serves a GGUF directly: @@ -801,9 +907,9 @@ async fn assert_lemonade_backend_alignment_opted_out(world: &mut E2eWorld) { /// (marked with `*`; see `ACTIVE_RUNTIME_MARKER` in `apps/rocm/src/main.rs`). /// /// Read independently of Lemonade's own reporting, on purpose: this is the -/// comparator [`assert_packaged_pin_survives`] uses to prove the pin was not -/// rewritten to match it, so it must not come from anything alignment itself -/// could produce (there is no `ROCm SDK:` line in `rocm version` on this +/// comparator [`assert_lemonade_backend_paired_with_active_sdk`] uses to prove +/// the pin was aligned to it, so it must not come from anything alignment +/// itself could produce (there is no `ROCm SDK:` line in `rocm version` on this /// branch -- that surface is a different, later change). fn active_runtime_version(world: &E2eWorld) -> String { let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]); @@ -821,12 +927,12 @@ fn active_runtime_version(world: &E2eWorld) -> String { /// The packaged pin was not rewritten. /// /// Reads `resources/backend_versions.json` inside the runtime tree the install -/// itself reports (`env_path:`) and compares it against the active runtime's -/// own version (from `rocm runtimes list`, not from anything Lemonade's -/// alignment reports), rather than inferring the outcome from an absent log -/// line alone. Both checks matter: the revert path can fail to restore the -/// pin (best-effort, matching its sibling warnings) while still printing no -/// "Aligned ..." line, so absence of that line alone cannot distinguish a +/// itself reports (`env_path:`) and compares it with the copy inside the +/// Lemonade archive the install verified, rather than inferring the outcome +/// from an absent log line alone. The absent-line checks matter too: the +/// revert path can fail to restore the pin (best-effort, matching its sibling +/// warnings) while still printing no "Aligned ..." line, so absence of that +/// line alone cannot distinguish a /// failed-restore from an honestly untouched pin -- hence also checking for /// the revert path's own "could not align ... reverting" warning, which does /// fire whenever a restore was attempted. `--reinstall` (the When's own @@ -849,34 +955,86 @@ async fn assert_packaged_pin_survives(world: &mut E2eWorld) { "an alignment attempt ran (and failed) instead of never starting:\n{stderr}" ); + let pinned_version = installed_therock_pin(output); + let packaged_version = packaged_therock_pin(output); + // Where the active SDK already is the packaged pin (a ROCm 10.1 SDK against + // Lemonade 2026.44.1) the ROCm version alone cannot tell an untouched pin + // from one rewritten to the same value, so the llama.cpp tag paired with it + // is checked too. + assert_eq!( + pinned_version, packaged_version, + "the pin differs from the one the Lemonade archive ships even though the user opted out" + ); + assert_eq!( + installed_backend_pin(output, "llamacpp", "rocm-stable"), + packaged_backend_pin(output, "llamacpp", "rocm-stable"), + "the llama.cpp ROCm tag differs from the one the Lemonade archive ships even though the user opted out" + ); +} + +/// `therock.version` as the Lemonade embeddable archive the install verified +/// ships it, read from the cached archive beside the runtime tree +/// (`/downloads/` next to `/runtime`) with the system `tar`, which +/// reads both the Linux `.tar.gz` and, as bsdtar on Windows, the `.zip`. The +/// archive is the one named by the install's `env_id` +/// (`lemonade-embeddable-`): the downloads directory is never pruned, +/// so a runner that installed an earlier pin still holds that archive too. +fn packaged_therock_pin(output: &str) -> String { + packaged_backend_pin(output, "therock", "version") + .unwrap_or_else(|| panic!("no therock.version in the packaged backend_versions.json")) +} + +/// `
.` as the cached Lemonade archive ships it; see +/// [`packaged_therock_pin`]. +fn packaged_backend_pin(output: &str, section: &str, key: &str) -> Option { let env_path = output .lines() .find_map(|line| line.trim().strip_prefix("env_path: ")) .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); - let backend_versions_path = - std::path::Path::new(env_path).join("resources/backend_versions.json"); - let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { - panic!( - "failed to read {}: {error}", - backend_versions_path.display() - ) - }); - let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { - panic!( - "failed to parse {}: {error}", - backend_versions_path.display() - ) - }); - let pinned_version = parsed["therock"]["version"] - .as_str() - .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())); - - let active_version = active_runtime_version(world); - assert_ne!( - pinned_version, active_version, - "the packaged pin was rewritten to the active ROCm SDK version even though the user \ - opted out" - ); + let env_id = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_id: ")) + .unwrap_or_else(|| panic!("no env_id in install output:\n{output}")); + let archive_prefix = format!("{env_id}-"); + let downloads = std::path::Path::new(env_path) + .parent() + .unwrap_or_else(|| panic!("env_path {env_path} has no parent")) + .join("downloads"); + // The archive's top-level directory is its file name without the + // extension. rocm-cli names the archive itself (`lemonade_archive_name`), + // always in lower case; the extension is compared ignoring ASCII case, as + // clippy's `case_sensitive_file_extension_comparisons` asks. + let archive_stem = |path: &std::path::Path| -> Option { + let name = path.file_name()?.to_str()?; + [".tar.gz", ".zip"].iter().find_map(|extension| { + let split = name.len().checked_sub(extension.len())?; + let (stem, tail) = (name.get(..split)?, name.get(split..)?); + (tail.eq_ignore_ascii_case(extension) && stem.starts_with(&archive_prefix)) + .then(|| stem.to_owned()) + }) + }; + let (archive, stem) = std::fs::read_dir(&downloads) + .unwrap_or_else(|error| panic!("failed to list {}: {error}", downloads.display())) + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find_map(|path| archive_stem(&path).map(|stem| (path, stem))) + .unwrap_or_else(|| panic!("no cached {env_id} archive in {}", downloads.display())); + let member = format!("{stem}/resources/backend_versions.json"); + let extracted = std::process::Command::new("tar") + .arg("-xOf") + .arg(&archive) + .arg(&member) + .output() + .unwrap_or_else(|error| panic!("failed to run tar on {}: {error}", archive.display())); + assert!( + extracted.status.success(), + "tar could not read {member} from {}: {}", + archive.display(), + String::from_utf8_lossy(&extracted.stderr) + ); + let parsed: serde_json::Value = serde_json::from_slice(&extracted.stdout) + .unwrap_or_else(|error| panic!("failed to parse {member}: {error}")); + parsed[section][key].as_str().map(str::to_owned) } /// The engine inventory reports a usable engine runtime. diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index 375a3dfa39..25cd34bef6 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -25,7 +25,7 @@ const ENGINE: &str = "vllm"; /// A record whose manifest a user already deleted by hand, leaving its engine /// state file stranded — the state this feature has to be able to clean up. const ORPHAN_ID: &str = "vllm-e2e-hand-deleted"; -/// A service whose `rocm serve` has written its 0600 endpoint key but not yet +/// A service whose `rocm serve` has written its owner-only endpoint key but not yet /// its record — the launch-in-progress shape the leftover sweep must not /// mistake for something to delete. const STARTING_ID: &str = "vllm-e2e-still-starting"; @@ -166,6 +166,31 @@ fn lists_live_server_row(listed: &str) -> bool { .any(|line| line.trim_end().strip_prefix("- ") == Some(SERVICE_ID)) } +/// Where a `rocm` save of `id`'s endpoint key that was killed between writing +/// the key and putting it in place leaves its copy: the shared atomic writer's +/// staged file, `{key file}.tmp-{pid}-{millis}-{attempt}`. +fn leftover_key_copy(world: &E2eWorld, id: &str) -> PathBuf { + services_dir(world).join(format!("{id}.endpoint-key.tmp-4242-1700000000000-0")) +} + +/// Plant that leftover copy, after checking the CLI's own reader takes the +/// name for a staged copy of `id`'s key — so a change to the writer's naming +/// fails here, as a premise, rather than as a mysterious survivor later. +fn plant_leftover_key_copy(world: &E2eWorld, id: &str) { + let copy = leftover_key_copy(world, id); + let name = copy + .file_name() + .and_then(|name| name.to_str()) + .expect("UTF-8 name"); + assert_eq!( + rocm_core::atomic_write::staged_destination_name(name), + Some(format!("{id}.endpoint-key").as_str()), + "premise: {name} must read as a staged copy of {id}'s key" + ); + std::fs::create_dir_all(services_dir(world)).expect("failed to create services dir"); + std::fs::write(©, "e2e-endpoint-key").expect("failed to write the leftover key copy"); +} + // ── Given ────────────────────────────────────────────────────────── #[given("a local server record that is no longer running")] @@ -230,6 +255,16 @@ async fn engine_state_cannot_be_deleted(world: &mut E2eWorld) { ); } +#[given("an interrupted save left a second copy of that server's API key")] +async fn leftover_copy_of_the_record_key(world: &mut E2eWorld) { + plant_leftover_key_copy(world, SERVICE_ID); +} + +#[given("an interrupted save left a copy of the API key of the record deleted by hand")] +async fn leftover_copy_of_the_orphan_key(world: &mut E2eWorld) { + plant_leftover_key_copy(world, ORPHAN_ID); +} + #[given("an engine state file whose local server record was deleted by hand")] async fn orphaned_engine_state(world: &mut E2eWorld) { let states = engine_state_dir(world); @@ -242,7 +277,7 @@ async fn orphaned_engine_state(world: &mut E2eWorld) { } /// A managed launch caught between its two writes, staged exactly as `rocm -/// serve` stages it: the 0600 endpoint key is on disk, the record is not, and +/// serve` stages it: the owner-only endpoint key is on disk, the record is not, and /// the managed-launch lock is held across the whole of that interval. /// /// The lock is taken with the production [`rocm_core::FileLock`] on the real @@ -547,7 +582,7 @@ async fn prune_did_not_return_before_the_hold(world: &mut E2eWorld) { ); } -/// The scenario's discriminating assertion: the 0600 key of a launch that has +/// The scenario's discriminating assertion: the owner-only key of a launch that has /// not published its record yet must survive the sweep. /// /// It discriminates only while `rocm`'s startup is shorter than the staged hold. @@ -583,6 +618,53 @@ async fn starting_key_present(world: &mut E2eWorld) { ); } +#[then("the second copy of that server's API key is gone")] +async fn record_key_copy_gone(world: &mut E2eWorld) { + let copy = leftover_key_copy(world, SERVICE_ID); + assert!( + !copy.exists(), + "{} holds the key and must be deleted with the record:\n{}", + copy.display(), + combined_output(world) + ); +} + +/// The four planted files plus the leftover copy: the summary's count is how +/// the user learns that more than the four usual files went. +#[then("the CLI counts that copy among the files it removed")] +async fn counts_the_key_copy(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!( + combined.contains("files removed: 5"), + "the removal must count the leftover key copy (4 record files + 1):\n{combined}" + ); +} + +#[then("the preview lists the leftover copy of the API key")] +async fn preview_lists_the_key_copy(world: &mut E2eWorld) { + let copy = leftover_key_copy(world, ORPHAN_ID); + let combined = combined_output(world); + assert!( + combined.contains(©.display().to_string()), + "the preview must name the leftover key copy {}:\n{combined}", + copy.display() + ); + assert!( + copy.exists(), + "a preview must not delete the leftover key copy" + ); +} + +#[then("the leftover copy of the API key is gone")] +async fn orphan_key_copy_gone(world: &mut E2eWorld) { + let copy = leftover_key_copy(world, ORPHAN_ID); + assert!( + !copy.exists(), + "the leftover key copy of a deleted record must be swept:\n{}", + combined_output(world) + ); +} + #[then("the leftover engine state file is gone")] async fn orphan_gone(world: &mut E2eWorld) { let orphan = engine_state_dir(world).join(format!("{ORPHAN_ID}.json")); diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 024f0b5fee..39d7e92d0e 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -4,7 +4,8 @@ //! Steps for stopping a managed service: `rocm services stop --yes` //! (confirmed, and unconfirmed through the `ROCM_E2E_STOP_OUTCOME` seam), and -//! `rocmd sandbox-tool stop_server`. +//! `rocmd sandbox-tool stop_server`; and for `engine_ipc.feature`, the steps +//! that run `rocm __engine-stdio vllm` with a request and read its answer. //! //! Black-box: plants a managed-service record as plain JSON in its own isolated //! data dir — the same on-disk shape `rocm serve --managed` writes — then runs @@ -16,6 +17,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use cucumber::{given, then, when}; +use rocm_engine_protocol::ENGINE_PROTOCOL_VERSION; use crate::E2eWorld; @@ -201,6 +203,15 @@ async fn managed_server_with_a_recycled_pid(world: &mut E2eWorld) { /// the process: it was not signalled, so it is still running. #[then("the process now holding the recorded process id is still running")] async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { + assert_recorded_process_still_running( + world, + "the stop signalled a process that is not the recorded server", + ); +} + +/// The process the scenario recorded has not exited; `failure` says what it +/// means if it has. +fn assert_recorded_process_still_running(world: &mut E2eWorld, failure: &str) { let process = world .recorded_process .as_mut() @@ -210,7 +221,7 @@ async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { .try_wait() .expect("failed to check the recorded process") .is_none(), - "the stop signalled a process that is not the recorded server" + "{failure}" ); } @@ -450,6 +461,75 @@ async fn engine_prints_garbage_and_fails(world: &mut E2eWorld) { ); } +/// The version a stand-in engine names when it answers in a protocol this +/// release cannot use: a later major, which no caret rule accepts. +const INCOMPATIBLE_PROTOCOL: &str = "9.0.0"; + +/// This release's engine protocol with its patch raised by one: a newer +/// protocol that only adds to this release's, which the caret rule accepts. +fn next_patch_version() -> String { + let (rest, patch) = ENGINE_PROTOCOL_VERSION + .rsplit_once('.') + .expect("the engine protocol version is MAJOR.MINOR.PATCH"); + let patch: u64 = patch.parse().expect("the patch is a number"); + format!("{rest}.{}", patch + 1) +} + +/// The stop answer every stand-in below gives, with `version` as its +/// `protocol_version` (none at all for `None`, as an engine older than the +/// field writes it). +fn stop_answer(version: Option<&str>) -> String { + let mut answer = serde_json::json!({"ok": true, "data": {"stopped": true, "graceful": true}}); + if let Some(version) = version { + answer["protocol_version"] = serde_json::Value::from(version); + } + answer.to_string() +} + +#[given("the engine answers the stop request in an engine protocol this release cannot use")] +async fn engine_answers_in_an_incompatible_protocol(world: &mut E2eWorld) { + install_standalone_daemon( + world, + &format!("echo '{}'", stop_answer(Some(INCOMPATIBLE_PROTOCOL))), + ); +} + +#[given("the engine writes a log line on stdout before its answer to the stop request")] +async fn engine_logs_on_stdout_before_answering(world: &mut E2eWorld) { + install_standalone_daemon( + world, + &format!( + "echo 'stand-in engine: loading model'\necho '{}'", + stop_answer(Some(ENGINE_PROTOCOL_VERSION)) + ), + ); +} + +/// An answer in this release's protocol, in a newer one that only added to it +/// (the next patch: same minor, so the caret rule accepts it), or with no +/// version, as an engine from before the field writes it. The newer one carries +/// the addition too: a field this release does not know, in the envelope and +/// in the stop's data, which it must ignore. +#[given( + regex = r"^the engine answers the stop request in (this release's engine protocol|a newer engine protocol that only adds to this release's|no named engine protocol)$" +)] +async fn engine_answers_in_a_usable_protocol(world: &mut E2eWorld, protocol: String) { + let next_patch = next_patch_version(); + let version = match protocol.as_str() { + "this release's engine protocol" => Some(ENGINE_PROTOCOL_VERSION), + "a newer engine protocol that only adds to this release's" => Some(next_patch.as_str()), + "no named engine protocol" => None, + other => panic!("no stand-in for {other:?}"), + }; + let mut answer: serde_json::Value = + serde_json::from_str(&stop_answer(version)).expect("the stop answer is JSON"); + if protocol == "a newer engine protocol that only adds to this release's" { + answer["hint"] = serde_json::Value::from("a field added after this release"); + answer["data"]["drained_requests"] = serde_json::Value::from(0); + } + install_standalone_daemon(world, &format!("echo '{answer}'")); +} + // ── When ─────────────────────────────────────────────────────────── /// Runs the pinned, directly-runnable form `serve_summary.rs`'s `stop` row @@ -595,9 +675,9 @@ async fn engine_answered_the_stop(world: &mut E2eWorld) { ); } -/// The stop report `rocmd sandbox-tool stop_server` printed, with the tool's own -/// exit status checked first: both engine endings below leave the stop itself -/// succeeding. +/// The stop report `rocmd sandbox-tool stop_server` printed, parsed as JSON, +/// after asserting that the tool exited 0: in every scenario that reads the +/// report this way, the stop itself succeeds whatever the engine answered. fn tool_report(world: &E2eWorld) -> serde_json::Value { let stdout = world.cli_output.as_deref().unwrap_or_default(); assert_eq!( @@ -634,12 +714,23 @@ async fn engine_received_one_stop_request(world: &mut E2eWorld) { .unwrap_or_else(|error| panic!("the request is not JSON ({error}): {requests:?}")); assert_eq!(request["method"], "stop", "{request}"); assert_eq!(request["payload"]["service_id"], SERVICE_ID, "{request}"); + // The daemon writes its request in its own engine protocol. + assert_eq!( + request["protocol_version"], ENGINE_PROTOCOL_VERSION, + "{request}" + ); } /// What the engine printed before it exited non-zero is what the report carries: /// a stop it answered, with no error taken from the exit status. #[then("the tool reports the engine's answer to the stop, not the status it exited with")] async fn engine_answer_survives_a_failing_exit(world: &mut E2eWorld) { + assert_engine_answered_the_stop(world); +} + +/// The report carries the stand-in's stop answer as the engine's result, and +/// the stop itself completed. +fn assert_engine_answered_the_stop(world: &E2eWorld) { let report = tool_report(world); assert_eq!( report["result"]["engine_stop"], @@ -661,13 +752,61 @@ async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { assert_eq!( engine_stop["error"], "failed to parse engine response envelope \ - (engine stdio process exited with code 1); \ + (engine stdio process exited with code 1): the engine's stdout is not JSON \ + (expected ident at line 1 column 2), starting with `this is not an answer` \ + (an engine writes only its JSON answer to stdout; logs go to stderr); \ stderr: engine exploded: no container runtime", "{report}" ); assert!(engine_stop.get("stopped").is_none(), "{report}"); } +/// An answer in a protocol this release cannot use is not taken as the stop's +/// result, even though it says the engine stopped: the report carries the +/// refusal, naming both versions, and nothing from the answer. +#[then( + "the tool reports the engine's answer refused, naming the engine's protocol and this release's" +)] +async fn incompatible_answer_refused(world: &mut E2eWorld) { + let report = tool_report(world); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!( + *engine_stop, + serde_json::json!({ + "attempted": true, + "error": format!( + "engine answered in engine protocol {INCOMPATIBLE_PROTOCOL}, which this \ + release (engine protocol {ENGINE_PROTOCOL_VERSION}) cannot use" + ), + }), + "{report}" + ); +} + +/// An answer with a log line before it on stdout is refused, and the error says +/// what the engine did wrong, quoting the line it wrote before its answer. +#[then("the tool reports the engine's answer refused for the log line the engine wrote before it")] +async fn contaminated_answer_refused(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!( + report["result"]["engine_stop"], + serde_json::json!({ + "attempted": true, + "error": "failed to parse engine response envelope: the engine wrote other \ + output to stdout before its answer, starting with `stand-in engine: \ + loading model` (an engine writes only its JSON answer to stdout; logs go \ + to stderr)", + }), + "{report}" + ); +} + +/// The engine's own answer is the stop's engine result. +#[then("the tool reports the engine's answer to the stop")] +async fn engine_answer_reported(world: &mut E2eWorld) { + assert_engine_answered_the_stop(world); +} + /// For a `vllm` server the engine's `Stop` is advisory: the recorded process /// decides, so a failed request does not leave the server unconfirmed. #[then("the tool still reports the server stopped")] @@ -900,3 +1039,204 @@ fn assert_succeeded(world: &E2eWorld) { combined_output(world) ); } + +// ── An engine reading a request (engine-ipc-08 to -12) ───────────── + +/// The `vllm` engine's own record of a server (`/engines/vllm/state/`), +/// as the engine writes it when it starts one. +fn vllm_engine_state_path(world: &E2eWorld) -> PathBuf { + engine_state_dir(world).join(format!("{SERVICE_ID}.json")) +} + +/// A process the `vllm` engine recorded as a server it started: a real `sleep`, +/// named in the engine's state file with the start-time token `/proc` gives it, +/// so a `stop` the engine served would find and end it. +#[given("an engine server whose process the vllm engine recorded is running")] +async fn vllm_engine_recorded_a_running_server(world: &mut E2eWorld) { + let process = Command::new("sleep") + .arg("300") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process to record"); + let state = serde_json::json!({ + "service_id": SERVICE_ID, + "status": "ready", + "pid": process.id(), + "start_ticks": start_ticks(process.id()), + }); + world.recorded_process = Some(process); + std::fs::create_dir_all(engine_state_dir(world)).expect("failed to create engine state dir"); + std::fs::write( + vllm_engine_state_path(world), + serde_json::to_vec_pretty(&state).expect("failed to serialize engine state"), + ) + .expect("failed to write the engine's state"); +} + +/// Runs `rocm __engine-stdio vllm`, the command the daemon reaches an engine +/// through, sends it `request` on stdin and records what it printed and how it +/// exited. +fn send_to_vllm_over_stdio(world: &mut E2eWorld, request: &str) { + use std::io::Write as _; + let mut command = Command::new(crate::rocm_binary()); + command + .args(["__engine-stdio", ENGINE]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + world.isolate_cmd(&mut command); + let mut child = command.spawn().expect("failed to run rocm __engine-stdio"); + child + .stdin + .take() + .expect("piped stdin") + .write_all(format!("{request}\n").as_bytes()) + .expect("failed to send the request"); + let output = child.wait_with_output().expect("failed to wait for rocm"); + world.cli_output = Some(String::from_utf8_lossy(&output.stdout).into_owned()); + world.cli_stderr = Some(String::from_utf8_lossy(&output.stderr).into_owned()); + world.cli_rc = Some(output.status.code().unwrap_or(-1)); +} + +/// A `stop` request for the server, written for [`INCOMPATIBLE_PROTOCOL`]. +#[when( + "the vllm engine is asked over stdio to stop that server, in an engine protocol it cannot serve" +)] +async fn ask_vllm_to_stop_in_an_incompatible_protocol(world: &mut E2eWorld) { + let request = serde_json::json!({ + "protocol_version": INCOMPATIBLE_PROTOCOL, + "method": "stop", + "payload": {"service_id": SERVICE_ID, "force": true}, + }); + send_to_vllm_over_stdio(world, &request.to_string()); +} + +/// The requests the engine-side scenarios send. `frobnicate` is a method no +/// engine has, so in an incompatible protocol the only answer that names the +/// protocol is one given before the method is decoded; the stop whose +/// version is the number `9` is likewise refused only if the version is read +/// before the envelope is decoded as a whole. +#[when( + regex = r"^the vllm engine is sent over stdio (a request for a method it does not have, in an engine protocol it cannot serve|a stop request for that server whose protocol version is not text|a request for a method it does not have, in its own engine protocol|something that is not a request)$" +)] +async fn send_vllm_a_request(world: &mut E2eWorld, request: String) { + let text = match request.as_str() { + "a request for a method it does not have, in an engine protocol it cannot serve" => { + serde_json::json!({ + "protocol_version": INCOMPATIBLE_PROTOCOL, + "method": "frobnicate", + "payload": {}, + }) + .to_string() + } + "a stop request for that server whose protocol version is not text" => serde_json::json!({ + "protocol_version": 9, + "method": "stop", + "payload": {"service_id": SERVICE_ID, "force": true}, + }) + .to_string(), + "a request for a method it does not have, in its own engine protocol" => { + serde_json::json!({ + "protocol_version": ENGINE_PROTOCOL_VERSION, + "method": "frobnicate", + "payload": {}, + }) + .to_string() + } + "something that is not a request" => "this is not a request".to_owned(), + other => panic!("no request for {other:?}"), + }; + send_to_vllm_over_stdio(world, &text); +} + +/// The answer `rocm __engine-stdio` printed, checked to be exactly an error +/// envelope with `code` and `message`, in this release's protocol, after a +/// clean exit: every refusal is an answer the daemon can report, not a crash. +fn assert_engine_error_answer(world: &E2eWorld, code: &str, message: &str) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "rocm __engine-stdio failed:\n{stdout}\n{}", + world.cli_stderr.as_deref().unwrap_or_default() + ); + let answer: serde_json::Value = + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")); + assert_eq!( + answer, + serde_json::json!({ + "protocol_version": ENGINE_PROTOCOL_VERSION, + "ok": false, + "error": {"code": code, "message": message}, + }), + "{answer}" + ); +} + +#[then( + "the engine answers that it cannot serve that engine protocol, naming its own and the request's" +)] +async fn engine_refused_the_protocol(world: &mut E2eWorld) { + assert_engine_error_answer( + world, + "incompatible_protocol", + &format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request was made \ + for protocol {INCOMPATIBLE_PROTOCOL}, which it cannot serve" + ), + ); +} + +#[then("the engine answers that it cannot read the request's protocol version")] +async fn engine_refused_an_unreadable_version(world: &mut E2eWorld) { + assert_engine_error_answer( + world, + "incompatible_protocol", + &format!( + "this engine speaks engine protocol {ENGINE_PROTOCOL_VERSION}; the request named \ + protocol version \"9\", which is not a MAJOR.MINOR.PATCH version, so it cannot be \ + served" + ), + ); +} + +#[then("the engine answers that it has no such method")] +async fn engine_has_no_such_method(world: &mut E2eWorld) { + assert_engine_error_answer( + world, + "unknown_method", + &format!( + "this engine (engine protocol {ENGINE_PROTOCOL_VERSION}) has no method \"frobnicate\"" + ), + ); +} + +#[then("the engine answers that what it was sent is not a request")] +async fn engine_says_not_a_request(world: &mut E2eWorld) { + assert_engine_error_answer( + world, + "invalid_request", + "the request is not JSON: expected ident at line 1 column 2", + ); +} + +#[then("the engine server's process is still running")] +async fn engine_server_still_running(world: &mut E2eWorld) { + assert_recorded_process_still_running( + world, + "the engine refused the stop but its server's process exited", + ); +} + +#[then("the vllm engine's record of that server still names its process")] +async fn vllm_engine_record_unchanged(world: &mut E2eWorld) { + let pid = recorded_pid(world); + let state: serde_json::Value = serde_json::from_slice( + &std::fs::read(vllm_engine_state_path(world)).expect("the engine's state is gone"), + ) + .expect("the engine's state is not JSON"); + assert_eq!(state["pid"], pid, "{state}"); + assert_eq!(state["status"], "ready", "{state}"); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9f5f4a6820..c649d4de6a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -52,9 +52,10 @@ const UNRELATED_LIVE_MODEL: &str = "e2e/unrelated-live-model"; /// [`REUSE_REFUSAL_MODEL`] picked it, so this aliases that constant rather than /// repeating the literal. const LEMONADE_SERVE_TARGET: &str = REUSE_REFUSAL_MODEL; -/// Model id for the OOM-launch scenario, kept distinct from every other e2e -/// serve model so its service records can never collide with another -/// scenario's. +/// Model id for the fabricated-launch scenarios (`@id:serve-oom-launch-memory-guidance` +/// and `@id:serve-smoke-failure-summary`), kept +/// distinct from every other e2e serve model so their service records can never +/// collide with another scenario's. const OOM_LAUNCH_MODEL: &str = "e2e/oom-launch-model"; /// How long to wait for a freshly served model's endpoint to become ready. /// @@ -94,9 +95,25 @@ pub(crate) async fn model_is_ready( expect_model: Option<&str>, timeout_secs: u64, ) -> bool { + model_is_ready_with_key(models_url, expect_model, timeout_secs, None).await +} + +/// [`model_is_ready`] against an endpoint protected by `api_key`, which a +/// keyless listing would only ever see refused. +async fn model_is_ready_with_key( + models_url: &str, + expect_model: Option<&str>, + timeout_secs: u64, + api_key: Option<&str>, +) -> bool { + let client = reqwest::Client::new(); let deadline = Instant::now() + Duration::from_secs(timeout_secs); while Instant::now() < deadline { - if let Ok(resp) = reqwest::get(models_url).await + let mut request = client.get(models_url); + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + if let Ok(resp) = request.send().await && resp.status().is_success() { match expect_model { @@ -627,6 +644,145 @@ async fn setup_gpu_model(world: &mut E2eWorld) { serve_gpu_model(world).await; } +/// The key the keyed-serve scenario protects its endpoint with. A fixed value: +/// what that scenario checks is whether the key is enforced, not how it is minted. +const PROTECTED_ENDPOINT_KEY: &str = "e2e-protected-endpoint-key-7f3a91c2"; + +#[given("a model is being served on GPU behind an API key")] +async fn setup_gpu_model_behind_key(world: &mut E2eWorld) { + serve_gpu_model_with_key(world, Some(PROTECTED_ENDPOINT_KEY)).await; +} + +/// Server routes outside the OpenAI `/v1` API that can run the model, read its +/// tokenizer or report on it. Each must demand the key like `/v1` does; the +/// bodies are valid requests, so a refusal can only come from authentication. +fn routes_outside_the_openai_api( + model: &str, +) -> Vec<(reqwest::Method, &'static str, serde_json::Value)> { + vec![ + ( + reqwest::Method::POST, + "/invocations", + serde_json::json!({ + "model": model, + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 8, + }), + ), + ( + reqwest::Method::POST, + "/tokenize", + serde_json::json!({"model": model, "prompt": "Hello"}), + ), + ( + reqwest::Method::POST, + "/detokenize", + serde_json::json!({"model": model, "tokens": [9707]}), + ), + (reqwest::Method::GET, "/metrics", serde_json::Value::Null), + ] +} + +/// The liveness probe, the one route the middleware answers without the key. +const HEALTH_ROUTE: &str = "GET /health"; + +/// Call every route in [`routes_outside_the_openai_api`] and then `GET /health`, +/// with or without the key, recording each status in `world.route_statuses`. +async fn call_routes_outside_the_openai_api(world: &mut E2eWorld, api_key: Option<&str>) { + let model = world.model_name.clone().expect("no model served"); + let client = reqwest::Client::builder() + .timeout(Duration::from_mins(2)) + .build() + .expect("http client"); + let base = format!("http://127.0.0.1:{SERVE_PORT}"); + world.route_statuses.clear(); + for (method, path, body) in routes_outside_the_openai_api(&model) { + let mut request = client.request(method.clone(), format!("{base}{path}")); + if !body.is_null() { + request = request.json(&body); + } + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + let status = request + .send() + .await + .unwrap_or_else(|e| panic!("{method} {base}{path} failed: {e}")) + .status() + .as_u16(); + world + .route_statuses + .push((format!("{method} {path}"), status)); + } + let mut health = client.get(format!("{base}/health")); + if let Some(key) = api_key { + health = health.bearer_auth(key); + } + let status = health + .send() + .await + .unwrap_or_else(|e| panic!("GET {base}/health failed: {e}")) + .status() + .as_u16(); + world.route_statuses.push((HEALTH_ROUTE.to_owned(), status)); +} + +#[when("a client calls the server's routes without the key")] +async fn client_calls_routes_without_key(world: &mut E2eWorld) { + call_routes_outside_the_openai_api(world, None).await; +} + +#[when("the client repeats those calls with the key")] +async fn client_repeats_calls_with_key(world: &mut E2eWorld) { + call_routes_outside_the_openai_api(world, Some(PROTECTED_ENDPOINT_KEY)).await; +} + +#[then("every route outside the OpenAI API refuses the call as unauthorized")] +async fn assert_other_routes_refused(world: &mut E2eWorld) { + let others: Vec<_> = world + .route_statuses + .iter() + .filter(|(route, _)| route != HEALTH_ROUTE) + .collect(); + assert!(!others.is_empty(), "no routes were called"); + let unrefused: Vec<_> = others + .into_iter() + .filter(|(_, status)| *status != 401) + .collect(); + assert!( + unrefused.is_empty(), + "routes answered a client without the key instead of 401: {unrefused:?}" + ); +} + +#[then("the health check answers without the key")] +async fn assert_health_answers(world: &mut E2eWorld) { + let health = world + .route_statuses + .iter() + .find(|(route, _)| route == HEALTH_ROUTE) + .expect("the health check was not called"); + assert!( + (200..300).contains(&health.1), + "{HEALTH_ROUTE} answered {}; the liveness probe must stay open", + health.1 + ); +} + +#[then("every one of those calls succeeds")] +async fn assert_all_routes_succeed(world: &mut E2eWorld) { + assert!(!world.route_statuses.is_empty(), "no routes were called"); + let failed: Vec<_> = world + .route_statuses + .iter() + .filter(|(_, status)| !(200..300).contains(status)) + .collect(); + assert!( + failed.is_empty(), + "routes refused a client presenting the key: {failed:?}" + ); +} + #[when("the user serves a model on GPU from the installed runtime")] async fn user_serves_gpu_model_from_installed_runtime(world: &mut E2eWorld) { // Unlike `setup_active_runtime`, this scenario intentionally keeps the @@ -637,6 +793,12 @@ async fn user_serves_gpu_model_from_installed_runtime(world: &mut E2eWorld) { } async fn serve_gpu_model(world: &mut E2eWorld) { + serve_gpu_model_with_key(world, None).await; +} + +/// Serve the host's GPU model, protecting the endpoint with `api_key` when one +/// is given (`--require-api-key`, so even this loopback bind demands it). +async fn serve_gpu_model_with_key(world: &mut E2eWorld, api_key: Option<&str>) { // Serve by the canonical HuggingFace ID (not the `qwen2.5` alias) with an // explicit engine matching this host. This step is a *precondition* for // scenarios that test inference/chat behavior, so it must not fail for @@ -675,12 +837,24 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm_bounded( + let mut args = vec!["serve", model, "--engine", engine, "--managed"]; + // The key goes through `ROCM_SERVE_API_KEY`, as `rocm serve --help` + // advises, not `--api-key`: argv is recorded verbatim in the uploaded + // `results/commands.jsonl`. + let mut envs = Vec::new(); + if let Some(key) = api_key { + args.push("--require-api-key"); + envs.push(("ROCM_SERVE_API_KEY", key)); + } + let (stdout, stderr, rc) = crate::run_rocm_bounded_with_env( world, - &["serve", model, "--engine", engine, "--managed"], + &args, + &envs, e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), ); - if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { + if rc == 0 + && model_is_ready_with_key(&MODELS_URL, Some(ready_substr), timeout_secs, api_key).await + { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); return; @@ -3220,6 +3394,173 @@ async fn assert_oom_launch_quotes_the_real_line(world: &mut E2eWorld) { ); } +/// A managed vLLM launch that became ready, whose engine (a mock) answers +/// `/v1/models` but fails every chat request with HTTP 500. The launch is +/// fabricated by the `ready` mode of the same test-only hook that +/// `@id:serve-oom-launch-memory-guidance` arms; the +/// mock's port is passed to `rocm serve` by the `When` below, which is how the +/// fabricated record points the post-launch smoke test at it. +#[given("a managed vLLM launch that becomes ready but whose engine fails its first chat request")] +async fn plant_ready_launch_with_failing_engine(world: &mut E2eWorld) { + // An empty chat script means this engine always fails chat: `/v1/models` + // answers, and every chat request is "past the end of the script" and gets + // HTTP 500. + let mock = MockServer::start_with_chat_script(OOM_LAUNCH_MODEL, Vec::new()).await; + world.mock = Some(mock); + // Consumed by `e2e_simulate_oom_launch` in + // `apps/rocm/src/service_launch/readiness.rs`; the name is spelled out on both + // sides because the binary's crate cannot depend on this harness. + world + .command_env + .push(("ROCM_E2E_SIMULATE_OOM_LAUNCH", "ready".into())); +} + +#[when("the user opens the interactive serve summary against that engine")] +async fn open_summary_against_failing_engine(world: &mut E2eWorld) { + let port = world + .mock + .as_ref() + .expect("no mock engine was started") + .port() + .to_string(); + let mut session = TuiSession::spawn( + world, + &[ + "serve", + OOM_LAUNCH_MODEL, + "--engine", + "vllm", + "--env-id", + "e2e-smoke-failure", + "--port", + &port, + ], + ) + .unwrap_or_else(|error| panic!("failed to open interactive serve summary: {error}")); + // `wait_for_exit` requires exit code 0: a failed smoke test must not fail + // the launch. + session + .wait_for_exit(interactive_summary_timeout()) + .await + .unwrap_or_else(|error| panic!("interactive serve summary failed: {error}")); + world.tui = Some(session); +} + +#[then("the deployment summary says the smoke test failed and does not call the service ready")] +async fn assert_summary_reports_failed_smoke_test(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains("Deploymentsummary(smoketestfailed)"), + "the heading must say the smoke test failed:\n{screen}" + ); + assert!( + flattened.contains("started,smoketestfailed"), + "the status row must not read `ready`:\n{screen}" + ); + assert!( + !flattened.contains("statusready"), + "a failed smoke test must not leave `status ready` on screen:\n{screen}" + ); +} + +/// Matches rocm's own wording of the status line (`the smoke test could not +/// complete`, `local provider returned ...`) around the engine's HTTP status, so +/// it pins that phrasing as well as the engine's reason. +#[then("the deployment summary gives the engine's own reason")] +async fn assert_summary_gives_engine_reason(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + // The exact reason the provider reports for the mock's answer, not just a + // `500` that the ephemeral port or a timestamp-based service id could also + // contain, and not the reason of some other failed outcome (for example "no + // ready managed service"). + assert!( + flattened.contains("thesmoketestcouldnotcomplete") + && flattened.contains("localproviderreturnedHTTP/1.1500InternalServerError"), + "the note must carry the engine's HTTP 500 as the reason:\n{screen}" + ); +} + +/// Passes with the fix reverted: the smoke request was sent before the fix too. +/// It proves only that the summary's request reached the mock once and was not +/// retried, so the failure the summary shows is that one request's. +#[then("the engine received exactly one chat request")] +async fn assert_engine_received_one_chat_request(world: &mut E2eWorld) { + let received = world + .mock + .as_ref() + .expect("no mock engine was started") + .chat_requests() + .len(); + assert_eq!( + received, 1, + "the smoke test must send exactly one chat request to the engine" + ); +} + +#[then("the deployment summary names the logs command for that service")] +async fn assert_summary_names_logs_command(world: &mut E2eWorld) { + let service_id = launched_service_record(world)["service_id"] + .as_str() + .expect("the service record has no service_id") + .to_owned(); + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains(&format!("check`rocmlogs--service{service_id}`")), + "the note must name `rocm logs --service {service_id}` in full:\n{screen}" + ); +} + +/// The only assertion behind the "ROCm CLI does not stop the server" promise +/// (README and `rocm serve --help`): a failed smoke test leaves the service +/// record `ready`. It passes with the fix reverted, so it does not check the +/// summary change; the steps above do. The exit code 0 is asserted in the `When` +/// step (`wait_for_exit`). +#[then("the launch's service record is left as ready")] +async fn assert_service_record_unchanged(world: &mut E2eWorld) { + assert_eq!( + launched_service_record(world)["status"], + "ready", + "a failed smoke test must not change the service record" + ); +} + +/// The one service record the fabricated launch wrote. +fn launched_service_record(world: &E2eWorld) -> serde_json::Value { + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let mut records: Vec = std::fs::read_dir(&services) + .expect("the launch left no services directory") + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .map(|entry| { + serde_json::from_slice(&std::fs::read(entry.path()).expect("record is readable")) + .expect("record is JSON") + }) + .collect(); + assert_eq!(records.len(), 1, "expected exactly the launched service"); + records.remove(0) +} + /// Collapse a rendered PTY screen to its non-whitespace characters, so an /// assertion survives the terminal soft-wrapping a long line across grid rows /// (which inserts row breaks mid-phrase / mid-token). Only meaningful for @@ -3823,20 +4164,25 @@ async fn serve_wait_refused(world: &mut E2eWorld) { ); } -/// The discriminating assertion for the fabricated-launch daemon suppression. +/// Hook hygiene, shared by the `@id:serve-oom-launch-memory-guidance` and +/// `@id:serve-smoke-failure-summary` scenarios: it records that no daemon was +/// started, and it does NOT discriminate the `if !e2e_simulate_oom_launch()` +/// guard in `serve_cmd.rs`. `ensure_background_helper_running_with` returns early +/// unless a watcher is enabled, and the isolated config these scenarios run in +/// enables none, so removing the guard leaves both scenarios green. /// -/// `simulate_oom_managed_launch` writes a `starting` record for a process that -/// never existed, so `ensure_background_helper_running_quiet` would leave a +/// `simulate_oom_managed_launch` writes a record (`starting`, or `ready` in its +/// `ready` mode) for a process that never existed, so +/// `ensure_background_helper_running_quiet` would leave a /// `rocm daemon` supervising a pid that will never appear — and nothing in this /// scenario's teardown stops it. Observed in the wild as daemons outliving their /// e2e checkout by days. /// -/// Without this step the suppression was untested: deleting the -/// `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs` left serve-41 green, -/// because every other assertion here is about the summary text. Both artefacts -/// the autostart path can leave are checked, since the daemon publishes its -/// runtime-state only after it is up — a spawn caught in the window before that -/// shows up as a claim and nothing else. +/// The guard would only matter in a config with a watcher enabled, which no +/// scenario here sets up; this step would catch a daemon in that case. Both +/// artefacts the autostart path can leave are checked, since the daemon +/// publishes its runtime-state only after it is up — a spawn caught in the +/// window before that shows up as a claim and nothing else. #[then("no background automation daemon was left behind")] async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let root = world @@ -3854,7 +4200,7 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { assert_ne!( state.get("running").and_then(serde_json::Value::as_bool), Some(true), - "a fabricated OOM launch must not start the automation daemon, but \ + "a fabricated launch must not start the automation daemon, but \ {} says one is running:\n{text}", state_path.display() ); @@ -3863,8 +4209,94 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let claim_path = root.join("autostart.claim"); assert!( !claim_path.is_file(), - "a fabricated OOM launch must not even claim a daemon autostart, but {} exists: {:?}", + "a fabricated launch must not even claim a daemon autostart, but {} exists: {:?}", claim_path.display(), std::fs::read_to_string(&claim_path).ok() ); } + +/// Serve this host's small model on loopback with `--require-api-key`, so the +/// CLI generates a key and stores it before the engine starts. The serve still +/// waits a fixed 45 seconds for the engine (`READINESS_WAIT` in +/// `apps/rocm/src/service_launch/readiness.rs`) and then exits 0, reporting a +/// server still loading as `starting`, but the check does not depend on +/// readiness: the key file is what the scenario checks, it exists from before +/// the spawn, and an engine still loading when the serve returns is not a +/// failure here. The World's teardown stops the service. +#[when("the user serves a model that requires an API key")] +async fn user_serves_model_requiring_api_key(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + ensure_serve_port_free().await; + // Bounded like `serve_and_wait`: a `rocm serve --managed` that never returns + // must fail this scenario, not stall the lane. + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--require-api-key", + ], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); + assert_eq!( + rc, 0, + "`rocm serve {model} --engine {engine} --managed --require-api-key` failed:\n\ + stdout:\n{stdout}\nstderr:\n{stderr}" + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// The one key file the serve wrote belongs to the service it recorded, holds +/// a key, and is mode 0600 exactly: no group or other bit, and nothing the +/// owner would need missing. +#[then("the server's API key is kept where only that user can read it")] +async fn endpoint_key_is_owner_only(world: &mut E2eWorld) { + let services_dir = world.app_paths().data_dir.join("services"); + let key_files: Vec = std::fs::read_dir(&services_dir) + .unwrap_or_else(|error| panic!("cannot read {}: {error}", services_dir.display())) + .map(|entry| entry.expect("services dir entry").path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "endpoint-key")) + .collect(); + assert_eq!( + key_files.len(), + 1, + "expected exactly one endpoint key file in {}, found {key_files:?}", + services_dir.display() + ); + let key_file = &key_files[0]; + let record = key_file.with_extension("json"); + assert!( + record.is_file(), + "the key file {} has no service record beside it", + key_file.display() + ); + let key = std::fs::read_to_string(key_file).expect("the owner can read the key back"); + assert!( + !key.trim().is_empty(), + "the key file {} is empty", + key_file.display() + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mode = std::fs::metadata(key_file) + .expect("key file metadata") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, + 0o600, + "the key file {} is mode {mode:o}, not 0600", + key_file.display() + ); + } + #[cfg(not(unix))] + panic!("this step checks a Unix file mode; its scenario is tagged @requires-os:linux"); +} diff --git a/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs index 10d3fd7e48..a3c4dc6384 100644 --- a/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs @@ -306,8 +306,9 @@ async fn told_how_to_end(world: &mut E2eWorld) { // The hint's own words. The end-of-input key it names is the one the When // step pressed, and the CLI finished because of it -- for the Linux key: // the scenario runs on Linux and WSL only, and the Windows key (Ctrl-Z then - // Enter) is checked by hand in docs/manual-testing.md section 13. The hint - // is wider than the terminal, so where it wraps is ignored. + // Enter) is checked by hand in docs/manual-testing.md's "Pasting An Error + // Into `rocm diagnose`" section. The hint is wider than the terminal, so + // where it wraps is ignored. assert!( squeeze(screen).contains(&squeeze( "Paste the error text, then end it with Ctrl-D at the start of a line \ diff --git a/tests/e2e-cucumber/tests/e2e/vllm_tensorizer_steps.rs b/tests/e2e-cucumber/tests/e2e/vllm_tensorizer_steps.rs new file mode 100644 index 0000000000..a878d04446 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/vllm_tensorizer_steps.rs @@ -0,0 +1,698 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `vllm_tensorizer.feature`: `rocm engines install vllm` against a +//! managed runtime whose vLLM pins a tensorizer that cannot be imported. +//! +//! The runtime's Python and `uv` are shell stand-ins that share state files in +//! the runtime folder, so the scenario can read what the "installed" tensorizer is +//! rather than trusting the CLI's account of it: +//! +//! - `tensorizer-version`: written by the stand-in `uv` when it installs +//! tensorizer (and reset to vLLM's pin when it installs vLLM), and seeded by +//! the scenario itself, as the already upgraded release, for the two scenarios +//! that plant a violation beside the upgraded tensorizer; +//! - `uv.log`: every `uv` invocation; +//! - `probe-ran`: written by the stand-in Python whenever the tensorizer import +//! probe runs; +//! - `repaired`: written by the stand-in `uv` when a vLLM install clears the +//! planted violation; +//! - marker files written when the scenario plants the runtime, which pick how +//! the stand-ins behave: `library-failure`, `library-failure-in-tensorizer`, +//! `key-error-in-torch`, `needs-pythonpath`, `needs-repair` and +//! `repair-needs-reinstall-flag`; +//! - `bin/vllm`, which the stand-in `uv` writes on every vLLM install (creating +//! it in a runtime that did not have it). +//! +//! The stand-ins use only shell builtins, because the scenario runs the CLI with +//! a PATH that holds nothing but its own shims. + +use std::path::{Path, PathBuf}; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +const RUNTIME_KEY: &str = "release-wheel-gfx1151-e2e-tensorizer"; + +/// The fixed release. Deliberately a literal rather than +/// `TENSORIZER_PYTHON314_FIX`: a bump of the constant must fail these scenarios +/// loudly so their stand-ins, which model that one release, are revisited. +const TENSORIZER_FIXED: &str = "2.12.1"; + +/// The line the CLI prints once it has replaced the unimportable pin. +const UPGRADE_WARNING: &str = "upgraded it to 2.12.1"; + +fn runtime_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("tensorizer-runtime") +} + +fn write_shim(path: &Path, body: &str) { + rocm_core::fake_executable::write_fake_executable(path, body) + .unwrap_or_else(|e| panic!("failed to write {}: {e}", path.display())); +} + +/// What installing tensorizer 2.12.1 does in the planted runtime. +#[derive(Clone, Copy)] +enum Upgrade { + /// The stand-in Python can import tensorizer afterwards. + Works, + /// `uv` succeeds but the stand-in Python still cannot import it. + NoEffect, + /// `uv` fails, as on an offline host. + Fails, +} + +/// What else is different about the planted runtime. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Extra { + /// Nothing: a runtime on a ROCm line without a discover build. + None, + /// A ROCm 10.1.0 discover runtime whose stand-in Python needs the `PYTHONPATH` + /// the server launches with, and, like a real one, does not see it under `-I`. + NeedsPythonPath, + /// The import fails inside torch, below a tensorizer frame, with a missing + /// library: not tensorizer's fault. + LibraryFailure, + /// The same missing library, raised by tensorizer's own code (the last frame + /// is in the tensorizer package): still not a Python-version failure. + LibraryFailureInTensorizer, + /// A `KeyError` raised in torch, below tensorizer frames: not a library-load + /// error, so only the last-frame rule can tell it is not tensorizer's. + KeyErrorInTorch, + /// A genuine unmet requirement beside the already upgraded tensorizer. A vLLM + /// install clears it and, like the real one, puts back vLLM's tensorizer pin. + /// Any `vllm==` install clears it here: the automatic repair does not pass + /// `--reinstall` (only the user's `--reinstall` does), and + /// [`Extra::ViolationClearedOnlyByReinstall`] models the violation a plain + /// install leaves. + ViolationBesideUpgrade, + /// As [`Extra::ViolationBesideUpgrade`], but the stand-in `uv` only clears the + /// violation when vLLM is installed with `--reinstall`, so the plain repair + /// leaves it and the CLI advises the reinstall. + ViolationClearedOnlyByReinstall, + /// vLLM is not installed in the runtime yet (no `vllm` command beside the + /// interpreter), so the install pass is a first install; the stand-in `uv` + /// creates the command when it is asked to install vLLM. + VllmNotInstalled, +} + +/// Plants the registry entry, the stand-in Python and `vllm` command, and the +/// stand-in `uv` and OpenMPI shims. `coloured` makes the stand-in Python write +/// its traceback with ANSI escapes around the quoted path, as Python 3.13+ does +/// under `FORCE_COLOR`. +fn plant(world: &E2eWorld, upgrade: Upgrade, coloured: bool, extra: Extra) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let rt = runtime_dir(world); + let bin = rt.join("bin"); + let shim = rt.join("shim"); + std::fs::create_dir_all(&bin).expect("create runtime bin"); + std::fs::create_dir_all(&shim).expect("create shim dir"); + let state = rt.display().to_string(); + + // vLLM's own probe answers `present`. The tensorizer import fails unless the + // version file says 2.12.1: with the real traceback's frame and exception when + // the server's launch environment is present, and with a torch-style library + // error when it is not (so a probe run without that environment would see a + // failure that is not tensorizer's and leave the runtime alone). + let traceback = if coloured { + r#"esc=$(printf '\033') + echo "${esc}[1;35mTraceback (most recent call last):${esc}[0m" >&2 + echo " File ${esc}[35m\"/venv/lib/python3.14/site-packages/tensorizer/_crypt/_encryption.py\"${esc}[0m, line 367, in __new__" >&2 + echo "${esc}[1;35mKeyError${esc}[0m: ${esc}[35m'crypto_stream_salsa20_NONCEBYTES'${esc}[0m" >&2"# + } else { + r#"echo 'File "/venv/lib/python3.14/site-packages/tensorizer/_crypt/_encryption.py", line 367, in __new__' >&2 + echo "KeyError: 'crypto_stream_salsa20_NONCEBYTES'" >&2"# + }; + write_shim( + &bin.join("python"), + &format!( + r#"#!/bin/sh +# Like a real interpreter, ignore PYTHONPATH under -E and under -I (which implies +# -E), alone or combined with other short flags (`-sE`), before the `-c` script; +# an option's value (`-Wignore::ImportWarning`) is not a flag. +for arg in "$@"; do + case "$arg" in + -c) break ;; + --*) ;; + # `-W` and `-X` take a value that may contain a capital E or I + # (`-Wignore::ImportWarning`, `-X importtime`): not the flags. + -W*|-X*) ;; + -*E*|-*I*) unset PYTHONPATH ;; + esac +done +case "$*" in + *tensorizer.serialization*) + : > "{state}/probe-ran" + if [ -e "{state}/library-failure" ] || [ -e "{state}/library-failure-in-tensorizer" ]; then + echo 'File "/venv/lib/python3.14/site-packages/tensorizer/serialization.py", line 12, in ' >&2 + if [ -e "{state}/library-failure" ]; then + echo 'File "/venv/lib/python3.14/site-packages/torch/__init__.py", line 300, in ' >&2 + fi + echo 'OSError: libamdhip64.so.7: cannot open shared object file: No such file or directory' >&2 + exit 1 + fi + if [ -e "{state}/key-error-in-torch" ]; then + echo 'File "/venv/lib/python3.14/site-packages/tensorizer/serialization.py", line 12, in ' >&2 + echo 'File "/venv/lib/python3.14/site-packages/torch/_utils.py", line 80, in helper' >&2 + echo "KeyError: 'gfx1151'" >&2 + exit 1 + fi + if [ -e "{state}/needs-pythonpath" ]; then + case ":$PYTHONPATH:" in + *":{state}/lib/python3.14/site-packages/_rocm_sdk_core/share/amd_smi:"*) ;; + *) + echo "ModuleNotFoundError: No module named 'amdsmi'" >&2 + exit 1 ;; + esac + fi + version="" + [ -e "{state}/tensorizer-version" ] && read -r version < "{state}/tensorizer-version" + if [ "$version" != "2.12.1" ]; then + if [ "$VLLM_TARGET_DEVICE" != rocm ]; then + echo 'OSError: libamdhip64.so.7: cannot open shared object file' >&2 + exit 1 + fi + {traceback} + exit 1 + fi ;; + *find_spec*) echo '{{"present": true, "version": "0.29.1"}}' ;; +esac +exit 0 +"# + ), + ); + if extra != Extra::VllmNotInstalled { + write_shim(&bin.join("vllm"), "#!/bin/sh\nexit 0\n"); + } + match extra { + Extra::None => {} + Extra::NeedsPythonPath => { + std::fs::create_dir_all(rt.join("lib").join("python3.14").join("site-packages")) + .expect("create site-packages"); + std::fs::write(rt.join("needs-pythonpath"), "").expect("write marker"); + } + Extra::LibraryFailure => { + std::fs::write(rt.join("library-failure"), "").expect("write marker"); + } + Extra::KeyErrorInTorch => { + std::fs::write(rt.join("key-error-in-torch"), "").expect("write marker"); + } + Extra::ViolationBesideUpgrade | Extra::ViolationClearedOnlyByReinstall => { + if extra == Extra::ViolationClearedOnlyByReinstall { + std::fs::write(rt.join("repair-needs-reinstall-flag"), "").expect("write marker"); + } + std::fs::write(rt.join("needs-repair"), "").expect("write marker"); + std::fs::write(rt.join("tensorizer-version"), "2.12.1\n").expect("write version"); + } + Extra::VllmNotInstalled => {} + Extra::LibraryFailureInTensorizer => { + std::fs::write(rt.join("library-failure-in-tensorizer"), "").expect("write marker"); + } + } + + // `pip check` reports vLLM's exact pin as unmet once 2.12.1 is installed, + // as the real `uv` does; installing tensorizer records the version only + // when the scenario says the upgrade helps. + let record = match upgrade { + Upgrade::Works => format!("echo 2.12.1 > \"{state}/tensorizer-version\""), + Upgrade::NoEffect => "true".to_owned(), + Upgrade::Fails => "echo 'error: network unreachable' >&2; exit 1".to_owned(), + }; + write_shim( + &shim.join("uv"), + &format!( + r#"#!/bin/sh +echo "$*" >> "{state}/uv.log" +case "$*" in + *"pip check"*) + failed="" + if [ -e "{state}/needs-repair" ] && [ ! -e "{state}/repaired" ]; then + echo "The package \`vllm\` requires \`numpy==2.0.0\`, but \`1.26.0\` is installed" >&2 + failed=1 + fi + version="" + [ -e "{state}/tensorizer-version" ] && read -r version < "{state}/tensorizer-version" + if [ "$version" = "2.12.1" ]; then + echo "The package \`vllm\` requires \`tensorizer==2.10.1\`, but \`2.12.1\` is installed" >&2 + failed=1 + fi + [ -z "$failed" ] || exit 1 ;; + *vllm==*) + printf '#!/bin/sh\nexit 0\n' > "{state}/bin/vllm" + echo 2.10.1 > "{state}/tensorizer-version" + if [ ! -e "{state}/repair-needs-reinstall-flag" ]; then + : > "{state}/repaired" + else + case "$*" in *--reinstall*) : > "{state}/repaired" ;; esac + fi ;; + *tensorizer==2.12.1*) {record} ;; +esac +exit 0 +"# + ), + ); + // OpenMPI reads as present, so the install does not reach for a package manager. + write_shim( + &shim.join("ldconfig"), + "#!/bin/sh\necho \"libmpi.so.40 (libc6,x86-64) => /usr/lib/libmpi.so.40\"\n", + ); + write_shim(&shim.join("mpirun"), "#!/bin/sh\nexit 0\n"); + + let registry = root.path().join("data").join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("create runtime registry"); + let manifest = serde_json::json!({ + "runtime_key": RUNTIME_KEY, + "runtime_id": RUNTIME_KEY, + "channel": "release", + "format": "wheel", + "family": "gfx1151", + "family_source": "e2e", + "version": if extra == Extra::NeedsPythonPath { "10.1.0" } else { "7.14.0" }, + "install_root": rt.display().to_string(), + "python_executable": bin.join("python").display().to_string(), + "selected_artifact_url": "https://example.invalid/e2e.whl", + "installed_at_unix_ms": 1, + }); + std::fs::write( + registry.join(format!("{RUNTIME_KEY}.json")), + serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), + ) + .expect("write runtime manifest"); +} + +fn install_vllm(world: &mut E2eWorld, extra: &[&str]) { + run_engines_install(world, &["install", "vllm"], extra); +} + +/// Runs `rocm engines --runtime-id --yes `. +fn run_engines_install(world: &mut E2eWorld, subcommand: &[&str], extra: &[&str]) { + let shim = runtime_dir(world).join("shim"); + let path = shim.display().to_string(); + let uv = shim.join("uv").display().to_string(); + let mut args = vec!["engines"]; + args.extend_from_slice(subcommand); + args.extend_from_slice(&["--runtime-id", RUNTIME_KEY, "--yes"]); + args.extend_from_slice(extra); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &args, + &[ + ("PATH", path.as_str()), + ("ROCM_CLI_UV_BINARY", uv.as_str()), + // A decoy that never satisfies the stand-in, and that replaces whatever + // PYTHONPATH the host exports: only the entry the server's launch + // environment prepends to it does, so the result does not depend on + // the host. + ("PYTHONPATH", "/nonexistent/decoy-pythonpath"), + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn combined(world: &E2eWorld) -> String { + format!( + "{}{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +fn uv_log(world: &E2eWorld) -> String { + std::fs::read_to_string(runtime_dir(world).join("uv.log")).unwrap_or_default() +} + +#[given("a managed vLLM runtime whose pinned tensorizer cannot be imported")] +async fn plant_fixable(world: &mut E2eWorld) { + plant(world, Upgrade::Works, false, Extra::None); +} + +#[given("a managed vLLM runtime whose Python writes its tracebacks in colour")] +async fn plant_coloured(world: &mut E2eWorld) { + plant(world, Upgrade::Works, true, Extra::None); +} + +#[given("a managed vLLM runtime whose tensorizer cannot be imported even after an upgrade")] +async fn plant_unfixable(world: &mut E2eWorld) { + plant(world, Upgrade::NoEffect, false, Extra::None); +} + +#[given("a managed vLLM runtime whose pinned tensorizer cannot be imported and cannot be upgraded")] +async fn plant_unupgradable(world: &mut E2eWorld) { + plant(world, Upgrade::Fails, false, Extra::None); +} + +#[given("a managed vLLM runtime that only imports with the server's PYTHONPATH")] +async fn plant_needs_pythonpath(world: &mut E2eWorld) { + plant(world, Upgrade::Works, false, Extra::NeedsPythonPath); +} + +#[given("a managed vLLM runtime whose import fails on a missing library below tensorizer")] +async fn plant_library_failure(world: &mut E2eWorld) { + plant(world, Upgrade::Works, false, Extra::LibraryFailure); +} + +#[given("a managed vLLM runtime whose import raises a non-library error in torch below tensorizer")] +async fn plant_key_error_in_torch(world: &mut E2eWorld) { + plant(world, Upgrade::Works, false, Extra::KeyErrorInTorch); +} + +#[given( + "a managed vLLM runtime whose violation beside the upgraded tensorizer only a reinstall clears" +)] +async fn plant_reinstall_only_violation(world: &mut E2eWorld) { + plant( + world, + Upgrade::Works, + false, + Extra::ViolationClearedOnlyByReinstall, + ); +} + +#[given("a managed vLLM runtime with a genuine violation beside the upgraded tensorizer")] +async fn plant_violation_beside_upgrade(world: &mut E2eWorld) { + plant(world, Upgrade::Works, false, Extra::ViolationBesideUpgrade); +} + +#[given( + "a managed runtime without vLLM whose pinned tensorizer cannot be imported and cannot be upgraded" +)] +async fn plant_first_install_unupgradable(world: &mut E2eWorld) { + plant(world, Upgrade::Fails, false, Extra::VllmNotInstalled); +} + +#[given("a managed vLLM runtime whose tensorizer code raises a missing-library error")] +async fn plant_library_failure_in_tensorizer(world: &mut E2eWorld) { + plant( + world, + Upgrade::Works, + false, + Extra::LibraryFailureInTensorizer, + ); +} + +#[given("the user has installed the vLLM engine from that runtime")] +#[when("the user installs the vLLM engine from that runtime")] +async fn user_installs_vllm(world: &mut E2eWorld) { + install_vllm(world, &[]); +} + +#[when("the user reinstalls the vLLM engine from that runtime")] +async fn user_reinstalls_vllm(world: &mut E2eWorld) { + install_vllm(world, &["--reinstall"]); +} + +#[then("the install succeeds")] +async fn install_succeeds(world: &mut E2eWorld) { + assert_eq!( + world.cli_rc, + Some(0), + "expected the install to succeed:\n{}", + combined(world) + ); +} + +#[then("the user is told tensorizer was upgraded to 2.12.1")] +async fn told_upgraded(world: &mut E2eWorld) { + let output = combined(world); + assert!( + output.contains(UPGRADE_WARNING) && output.contains("tensorizer"), + "expected the upgrade warning:\n{output}" + ); +} + +#[then("the dependency check reports the tensorizer pin as an expected divergence")] +async fn expected_divergence(world: &mut E2eWorld) { + let output = combined(world); + assert!( + output.contains("dependency_check: expected_divergence") + && output.contains("requires `tensorizer==2.10.1`, but `2.12.1` is installed"), + "expected the tensorizer pin as an expected divergence:\n{output}" + ); + // The block explains itself, so a repeat install (which prints no upgrade + // warning) still says why, and the action line points at that note. + // The note names the release the runtime holds, read from the state the + // stand-in `uv` recorded rather than from a constant. + let held = + std::fs::read_to_string(runtime_dir(world).join("tensorizer-version")).unwrap_or_default(); + assert_eq!(held.trim(), "2.12.1", "the runtime must hold the upgrade"); + assert!( + output.contains(&format!("this runtime has tensorizer {},", held.trim())) + && output + .contains("action: none; this tensorizer is kept on purpose (see the note above)"), + "expected the divergence block to carry its own reason and action:\n{output}" + ); + assert!( + !output.contains("dependency_check: violated") + && !output.contains("rocm engines install vllm --reinstall"), + "the deliberate divergence must not read as a violation or advise a reinstall:\n{output}" + ); +} + +#[then("tensorizer 2.12.1 is what the runtime now holds")] +async fn holds_fixed_tensorizer(world: &mut E2eWorld) { + let version = + std::fs::read_to_string(runtime_dir(world).join("tensorizer-version")).unwrap_or_default(); + assert_eq!(version.trim(), "2.12.1", "uv log:\n{}", uv_log(world)); + assert!( + uv_log(world).contains("--no-deps tensorizer==2.12.1"), + "the upgrade must leave vLLM's other dependencies alone:\n{}", + uv_log(world) + ); +} + +#[then("tensorizer was upgraded only once and vLLM was not reinstalled")] +async fn upgraded_once(world: &mut E2eWorld) { + let log = uv_log(world); + let upgrades = log + .lines() + .filter(|line| line.contains("tensorizer==2.12.1")) + .count(); + assert_eq!(upgrades, 1, "uv log:\n{log}"); + assert!( + !log.contains("vllm") && !log.contains("--reinstall"), + "no vLLM install or reinstall may run on the second install:\n{log}" + ); + assert!( + !combined(world).contains(UPGRADE_WARNING), + "the second install must not report a second upgrade:\n{}", + combined(world) + ); +} + +#[then("the install warns that tensorizer still cannot be imported and shows the KeyError")] +async fn warns_still_broken(world: &mut E2eWorld) { + let output = combined(world); + assert!( + output.contains("tensorizer still cannot be imported after upgrading") + && output.contains("KeyError: 'crypto_stream_salsa20_NONCEBYTES'"), + "expected the warning to name the failure:\n{output}" + ); + // No vLLM install ran on this runtime, so the warning must not claim one did, + // and the upgrade was really attempted. + assert!( + !output.contains("was installed"), + "an existing runtime's warning must not claim an install ran:\n{output}" + ); + let log = uv_log(world); + assert!( + log.contains("--no-deps tensorizer==2.12.1") && !log.contains("vllm=="), + "the upgrade is attempted and no vLLM install runs, uv log:\n{log}" + ); +} + +#[then("the install warns that tensorizer could not be upgraded")] +async fn warns_not_upgraded(world: &mut E2eWorld) { + let output = combined(world); + assert!( + output.contains("upgrading it to tensorizer==2.12.1 failed") + && output.contains("network unreachable") + && output.contains("vLLM would fail at startup"), + "expected the failed-upgrade warning:\n{output}" + ); + assert!( + !runtime_dir(world).join("tensorizer-version").exists(), + "a failed upgrade must leave the runtime's tensorizer as it was" + ); +} + +#[then("the install is refused because tensorizer could not be upgraded")] +async fn refused_not_upgraded(world: &mut E2eWorld) { + assert_ne!(world.cli_rc, Some(0), "{}", combined(world)); + let output = combined(world); + assert!( + output.contains("upgrading it to tensorizer==2.12.1 failed") + && output.contains("network unreachable"), + "expected the refusal to name the failed upgrade:\n{output}" + ); + // This run installed vLLM (a `--reinstall` or a first install) before the check, + // so the refusal may say so. + assert!( + output.contains("vLLM was installed, but"), + "a refusal after this run installed vLLM should say the install ran:\n{output}" + ); + assert!( + uv_log(world).contains("vllm"), + "this run must have installed vLLM, uv log:\n{}", + uv_log(world) + ); +} + +#[then("the install succeeds without changing tensorizer")] +async fn succeeds_without_change(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{}", combined(world)); + let output = combined(world); + assert!( + !output.contains(UPGRADE_WARNING) && !uv_log(world).contains("tensorizer=="), + "a failure that is not tensorizer's must leave it alone:\n{output}\n{}", + uv_log(world) + ); + assert!( + !runtime_dir(world).join("tensorizer-version").exists(), + "tensorizer must not have been touched" + ); + assert!( + runtime_dir(world).join("probe-ran").exists(), + "the probe must have looked at tensorizer, or nothing was left alone on purpose" + ); +} + +#[then( + "the refusal says the install ran and tensorizer still cannot be imported after the upgrade" +)] +async fn refusal_after_install_and_upgrade(world: &mut E2eWorld) { + assert_ne!(world.cli_rc, Some(0), "{}", combined(world)); + let output = combined(world); + assert!( + output.contains( + "vLLM was installed, but tensorizer still cannot be imported after upgrading" + ), + "expected the fatal message:\n{output}" + ); + let log = uv_log(world); + let install = log.find("vllm==").expect("the reinstall ran"); + let upgrade = log + .find("--no-deps tensorizer==2.12.1") + .expect("the upgrade was attempted"); + assert!(install < upgrade, "the upgrade follows the install:\n{log}"); +} + +#[then("the violation is cleared and tensorizer is upgraded again after vLLM was reinstalled")] +async fn violation_cleared_and_tensorizer_kept(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{}", combined(world)); + let output = combined(world); + assert!( + output.contains("dependency_check: expected_divergence"), + "the repair must leave only the divergence:\n{output}" + ); + // The repair warning above the check names the violation it found; the check + // itself, after the install, must no longer. + let (before, after) = output + .split_once("dependency_check:") + .expect("the dependency check ran"); + assert!( + before.contains("violation: The package `vllm` requires `numpy") + && !after.contains("numpy"), + "the violation must be found by the repair and gone from the check:\n{output}" + ); + let held = + std::fs::read_to_string(runtime_dir(world).join("tensorizer-version")).unwrap_or_default(); + assert_eq!( + held.trim(), + TENSORIZER_FIXED, + "the reinstall put vLLM's pin back, so tensorizer must have been upgraded again" + ); + let log = uv_log(world); + let install = log.find("vllm==").expect("vLLM was reinstalled"); + let upgrade = log + .find("--no-deps tensorizer==2.12.1") + .expect("tensorizer was upgraded"); + assert!(install < upgrade, "the upgrade follows the install:\n{log}"); +} + +/// The command the CLI printed beside the violation, as the `action:` line gives it. +fn advised_command(world: &E2eWorld) -> Vec { + let output = combined(world); + let line = output + .lines() + .find_map(|line| line.trim().strip_prefix("action: rocm ")) + .unwrap_or_else(|| panic!("the CLI printed no `action: rocm ...` advice:\n{output}")); + line.split_whitespace().map(str::to_owned).collect() +} + +#[then("the install reports the violation and advises reinstalling vLLM")] +async fn reports_violation_and_advises_reinstall(world: &mut E2eWorld) { + let output = combined(world); + assert!( + output.contains("dependency_check: violated") + && output.contains("violation: The package `vllm` requires `numpy") + && output.contains("divergence: The package `vllm` requires `tensorizer"), + "expected the violation beside the divergence:\n{output}" + ); + let advice = advised_command(world); + assert_eq!( + advice.first().map(String::as_str), + Some("engines"), + "{advice:?}" + ); + // The plain install left the violation: the repair did not clear it. + assert!( + !runtime_dir(world).join("repaired").exists(), + "the stand-in clears the violation only on --reinstall" + ); +} + +#[when("the user runs the advised command")] +async fn runs_advised_command(world: &mut E2eWorld) { + let advice = advised_command(world); + let words: Vec<&str> = advice.iter().map(String::as_str).collect(); + // The advice names the subcommand and flags; the planted runtime id and the + // approval are what a user would add for this machine. + let split = words + .iter() + .position(|part| part.starts_with("--")) + .unwrap_or(words.len()); + let (subcommand, flags) = words.split_at(split); + run_engines_install(world, &subcommand[1..], flags); +} + +#[then("the advised reinstall cleared the violation and tensorizer was upgraded after it")] +async fn advised_reinstall_worked(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{}", combined(world)); + let output = combined(world); + let (_, after) = output + .split_once("dependency_check:") + .expect("the dependency check ran"); + assert!( + output.contains("dependency_check: expected_divergence") && !after.contains("numpy"), + "the reinstall must clear the violation and leave only the divergence:\n{output}" + ); + assert!(runtime_dir(world).join("repaired").exists()); + let held = + std::fs::read_to_string(runtime_dir(world).join("tensorizer-version")).unwrap_or_default(); + assert_eq!(held.trim(), TENSORIZER_FIXED); + let log = uv_log(world); + let reinstall = log + .lines() + .position(|line| line.contains("vllm==") && line.contains("--reinstall")) + .unwrap_or_else(|| panic!("vLLM was never reinstalled:\n{log}")); + let upgrade = log + .lines() + .enumerate() + .filter(|(_, line)| line.contains("--no-deps tensorizer==2.12.1")) + .map(|(index, _)| index) + .last() + .expect("tensorizer was upgraded"); + assert!( + upgrade > reinstall, + "tensorizer must be upgraded after the reinstall put vLLM's pin back:\n{log}" + ); +} diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 47dcb5d9d6..451234c6f5 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -61,6 +61,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("therock_next_generation.feature", "therock-next"), ("uninstall.feature", "uninstall"), ("update.feature", "update"), + ("vllm_tensorizer.feature", "vllm-tensorizer"), ]; fn features_dir() -> PathBuf { diff --git a/xtask/src/ascii_output_contract.rs b/xtask/src/ascii_output_contract.rs index 18827f11d6..d4d802aef1 100644 --- a/xtask/src/ascii_output_contract.rs +++ b/xtask/src/ascii_output_contract.rs @@ -162,6 +162,12 @@ mod tests { "The `©` is in a `#` shell comment of the installer `rocm remote \ provision` uploads. The shell never echoes a comment.", ), + ( + "engines/vllm/src/rocm_cli_vllm_auth.py", + "The `©` is in the `#` copyright comment of the auth middleware \ + module `rocm serve` writes for vLLM to import. Python never prints \ + a comment.", + ), ]; /// Individual literals that are out of scope, as diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 1d8cd0e531..3b376ed73f 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -106,9 +106,10 @@ pub fn run(args: &[String]) -> Result<()> { if binaries.build_release { let status = Command::new(&cargo) // `--features rocm/e2e-oom-fault-injection` compiles in the test-only - // `rocm serve` OOM-launch fault injection used by the positive - // `serve-oom-launch-memory-guidance` scenario. It is scoped to the - // `rocm` package (rocmd has no such feature) and to this e2e build + // `rocm serve` launch fault injection (an OOM launch, or a ready + // launch) used by the scenarios tagged `@requires-oom-fault-injection`. + // It is scoped to the `rocm` package (rocmd has no such feature) and + // to this e2e build // only — the release build never enables it, so shipped binaries // carry no fault-injection hook. It joins `rocm/e2e-test-hooks` // (the general e2e hook surface) in a single space-separated diff --git a/xtask/src/e2e_prewarm.rs b/xtask/src/e2e_prewarm.rs index abd07a44cc..bf48d19c67 100644 --- a/xtask/src/e2e_prewarm.rs +++ b/xtask/src/e2e_prewarm.rs @@ -53,6 +53,7 @@ use std::process::Command; use anyhow::{Context, Result, bail}; +use crate::e2e_prewarm_vision::{repair_torchvision_mismatch, verify_runtime_imports}; use crate::paths::{binary_name, release_binary_dir, workspace_root}; /// What the pre-warm should do with the shared tree, given the current @@ -338,6 +339,19 @@ pub fn assess(runtimes_list_report: &str, runtimes_dir: &Path) -> Vec bool { + update_report + .lines() + .filter_map(RuntimeLine::parse) + .any(|line| { + line.runtime_key == runtime_key && line.status.as_deref() == Some("ahead_of_index") + }) +} + /// One ` {marker} runtime_id=… format=… mode=… status=…` line from /// `rocm runtimes list`. /// @@ -430,8 +444,8 @@ pub fn dangling_active_runtime(runtimes_list_report: &str) -> Option<&str> { }) } -/// The already-installed runtime (from `rocm runtimes list`, which carries no -/// `channel=` field at all) whose key names `pin` within `channel`. +/// The already-installed runtime (from `rocm runtimes list`) whose key names +/// `pin` within `channel`. /// /// Mirrors `decide()`'s pin branch, which scopes its own search to /// `channel_runtimes` (or a same-channel unattributed error line) before ever @@ -439,9 +453,11 @@ pub fn dangling_active_runtime(runtimes_list_report: &str) -> Option<&str> { /// pin's token/digit-run appears in a key, with no notion of channel, so an /// unscoped search here could match a `release` pin against an installed /// `nightly` runtime that happens to share the pin's version/build-date -/// digits, silently serving the wrong channel's SDK. Since this listing has no -/// channel field to key off at all, the guard is the same `{channel}-` prefix -/// `decide()` falls back to for its own degraded, channel-less lines. +/// digits, silently serving the wrong channel's SDK. The listing prints +/// `channel=` only on each runtime's separate `provenance:` line, and +/// `RuntimeEntry::parse` reads only the row line, so the guard is the same +/// `{channel}-` key prefix `decide()` falls back to for its own degraded, +/// channel-less lines. /// /// Read-only (adopted/imported) entries are excluded for the same reason /// `activation_candidates` excludes them: a matching entry here becomes @@ -662,6 +678,11 @@ pub fn run( // Repairing first means `decide` reads a registry with nothing dead in it. repair_poisoned_runtimes(&rocm, prewarm_dir)?; + // Likewise a runtime whose torch and torchvision no longer match: remove it so + // `decide` below reads a registry without it. Before the dangling-pointer + // repair, which cleans up after this removal as it does after the one above. + let repair_declined = repair_torchvision_mismatch(&rocm, prewarm_dir, pin.is_some())?; + // And point the tree at something that exists before anything resolves the // active runtime. Must run after the repair above, which can itself remove // the runtime the pointer names. @@ -795,7 +816,17 @@ pub fn run( // runtime with no engine, which is the one thing those lanes exist to // exercise. Re-checking a warm tree is cheap: without `--reinstall`, // `engines install` on a ready engine installs nothing. - ensure_default_engine(&rocm, prewarm_dir)?; + let engine = ensure_default_engine(&rocm, prewarm_dir)?; + + // Also unconditional and before the reuse return, for the same reason: a + // persisted runtime can be current by version and still unusable. + verify_runtime_imports( + &rocm, + prewarm_dir, + runtime_changed(&decision), + &engine, + repair_declined, + )?; if !runtime_changed(&decision) { return Ok(()); @@ -854,7 +885,20 @@ const fn runtime_changed(decision: &Decision) -> bool { /// path: reusing a stale-but-working runtime keeps a lane meaningful, whereas /// serving with no engine fails every GPU scenario later and for reasons that /// name none of this. -fn ensure_default_engine(rocm: &Path, prewarm_dir: &Path) -> Result<()> { +/// +/// Returns the engine it ensured, so the checks after it can ask what that engine +/// needs of the runtime. +fn ensure_default_engine(rocm: &Path, prewarm_dir: &Path) -> Result { + let engine = default_engine(rocm, prewarm_dir)?; + println!("pre-warm: ensuring the {engine} engine is installed"); + rocm_command(rocm, prewarm_dir) + .args(["engines", "install", &engine, "--yes"]) + .status_ok("rocm engines install")?; + Ok(engine) +} + +/// The engine `rocm engines list` marks as the default for this host. +pub(crate) fn default_engine(rocm: &Path, prewarm_dir: &Path) -> Result { let output = rocm_command(rocm, prewarm_dir) .args(["engines", "list"]) .output() @@ -863,12 +907,9 @@ fn ensure_default_engine(rocm: &Path, prewarm_dir: &Path) -> Result<()> { bail!("`rocm engines list` exited with {}", output.status); } let inventory = String::from_utf8_lossy(&output.stdout); - let engine = default_engine_from_inventory(&inventory) - .context("`rocm engines list` did not identify a default engine")?; - println!("pre-warm: ensuring the {engine} engine is installed"); - rocm_command(rocm, prewarm_dir) - .args(["engines", "install", engine, "--yes"]) - .status_ok("rocm engines install") + default_engine_from_inventory(&inventory) + .map(str::to_owned) + .context("`rocm engines list` did not identify a default engine") } /// The engine `rocm engines list` marks as the default for this host, if any. @@ -1050,7 +1091,7 @@ fn list_runtimes(rocm: &Path, prewarm_dir: &Path) -> Result { /// Ask the CLI whether the installed runtimes are current. Check-only: plain /// `rocm update` without `--apply` never mutates state. -fn probe(rocm: &Path, prewarm_dir: &Path) -> Result { +pub(crate) fn probe(rocm: &Path, prewarm_dir: &Path) -> Result { let output = rocm_command(rocm, prewarm_dir) .arg("update") .output() @@ -1070,7 +1111,7 @@ fn probe(rocm: &Path, prewarm_dir: &Path) -> Result { /// The three `ROCM_CLI_*` directories are what make this a SHARED tree rather /// than the caller's own; `HF_HOME` / `UV_CACHE_DIR` are inherited from the job /// environment, which the lanes already export. -fn rocm_command(rocm: &Path, prewarm_dir: &Path) -> Command { +pub(crate) fn rocm_command(rocm: &Path, prewarm_dir: &Path) -> Command { let mut cmd = Command::new(rocm); cmd.env("ROCM_CLI_CONFIG_DIR", prewarm_dir.join("config")) .env("ROCM_CLI_DATA_DIR", prewarm_dir.join("data")) @@ -1080,7 +1121,7 @@ fn rocm_command(rocm: &Path, prewarm_dir: &Path) -> Command { /// Run a command for its exit status, turning a non-zero exit into an error that /// names the command. -trait StatusOk { +pub(crate) trait StatusOk { fn status_ok(&mut self, what: &str) -> Result<()>; } @@ -1170,11 +1211,14 @@ source: index\n" ) } - /// A real `rocm runtimes list` on a tree holding all three shapes that matter, - /// captured verbatim from the built binary rather than written by hand: one - /// poisoned managed runtime (an install root under a per-scenario temp dir that - /// is gone), one healthy managed runtime inside the tree, and one read-only - /// runtime adopted from outside it. + /// A `rocm runtimes list` on a tree holding all three shapes that matter, + /// captured from the built binary rather than written by hand: one poisoned + /// managed runtime (an install root under a per-scenario temp dir that is + /// gone), one healthy managed runtime inside the tree, and one read-only + /// runtime adopted from outside it. The capture predates the rows' + /// `toolchain=` field, which is not added back here; the `provenance:` line + /// the listing now prints after each `install_root:` was added by hand, in + /// that position, so [`assess`] is read against the current row order. /// /// Note that ALL THREE report `status=unusable` here. That is the whole reason /// [`assess`] keys off the install root instead: the healthy one is unusable @@ -1192,10 +1236,13 @@ registered ROCm runtimes adopted-external-env runtime_id=external-adopted version=7.14.0 format=wheel family=gfx94X-dcgpu mode=read-only status=unusable (pip runtime manifest is missing rocm_sdk probe data) install_root: /opt/external-rocm + provenance: channel=release source=unrecorded source_layout_generation=unrecorded device_target=unrecorded release-wheel-gfx94x-dcgpu-7-15-0 runtime_id=therock-release-gfx94x-dcgpu version=7.15.0 format=wheel family=gfx94X-dcgpu mode=managed status=unusable (pip runtime manifest is missing rocm_sdk probe data) install_root: /w/e2e-prewarm/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-15-0 + provenance: channel=release source=https://example.invalid/whl source_layout_generation=unrecorded device_target=unrecorded release-wheel-gfx94x-dcgpu-7-13-0 runtime_id=therock-release-gfx94x-dcgpu version=7.13.0 format=wheel family=gfx94X-dcgpu mode=managed status=unusable (install root is missing: /tmp/rocm-e2e-7MidR2/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0) install_root: /tmp/rocm-e2e-7MidR2/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0 + provenance: channel=release source=https://example.invalid/whl source_layout_generation=unrecorded device_target=unrecorded "; /// A real `rocm runtimes list` on an empty tree, captured from the binary. @@ -1210,9 +1257,10 @@ registered ROCm runtimes next step: rocm install sdk --channel release --format wheel "; - /// A real `rocm runtimes list` on the tree that killed the lane: the config - /// still names the 7.14.0 runtime that was removed out of band, while two - /// runtimes it could be pointed at sit right there. + /// A `rocm runtimes list` on the tree that killed the lane, captured before + /// the rows gained `toolchain=` and `provenance:` lines, neither added here: + /// the config still names the 7.14.0 runtime that was removed out of band, + /// while two runtimes it could be pointed at sit right there. /// /// Note there is no `*` marker on any entry — the renderer only marks the /// active one by matching `active_runtime_key` against a manifest, and that is @@ -1374,11 +1422,12 @@ registered ROCm runtimes #[test] fn pinned_runtime_entry_does_not_match_another_channels_installed_runtime() { // 7.14.0 is only installed under `nightly`, not the requested - // `release`. `rocm runtimes list` carries no `channel=` field at all - // (unlike `rocm update`'s report), so the `{channel}-` prefix guard is - // the only thing standing between this and silently serving the wrong - // channel's SDK for a pin that happens to share another channel's - // installed version. + // `release`. `rocm runtimes list` prints `channel=` only on each + // runtime's `provenance:` line, which `RuntimeEntry::parse` does not + // read (it reads the row line alone), so the `{channel}-` key prefix + // guard is the only thing standing between this and silently serving + // the wrong channel's SDK for a pin that happens to share another + // channel's installed version. assert!(find_pinned_runtime_entry(MULTI_CHANNEL, "release", "7.14.0").is_none()); assert_eq!( find_pinned_runtime_entry(MULTI_CHANNEL, "nightly", "7.14.0").map(|e| e.runtime_key), @@ -1455,6 +1504,19 @@ mode=managed status=ready\n install_root: /tmp/rocm-e2e-XXXX/data/runtimes/ assert!(RuntimeEntry::parse(" install_root: /tmp/x").is_none()); } + #[test] + fn a_provenance_line_is_not_read_as_an_entry() { + // `rocm runtimes list` prints a `provenance:` line under each runtime, + // whose `name=value` fields sit beside the row's own. + assert!( + RuntimeEntry::parse( + " provenance: channel=release source=unrecorded \ +source_layout_generation=unrecorded device_target=unrecorded" + ) + .is_none() + ); + } + #[test] fn a_tarball_runtime_reports_its_own_format() { // The folder to remove is `//`, so the format has to @@ -2020,4 +2082,44 @@ message=connect timed out after 30 s"; // …and end to end, the real empty report still resolves to a cold install. assert_eq!(decide(EMPTY, "release", None), Decision::Install); } + + /// `run` is never called by a test, so what keeps the checks where the + /// comments on them say is this reading of `run`'s own code: comment lines + /// are dropped, and each call must appear in order. The repair comes before + /// the freshness decision and the dangling-pointer repair; the check comes + /// after the engine check and before the reuse return, which a warm tree + /// takes every time. Formatting and unrelated edits do not move these. + #[test] + fn ahead_of_index_is_read_for_the_named_runtime_only() { + let report = "runtime a channel=release status=ahead_of_index\n\ + runtime b channel=release status=up_to_date\n"; + assert!(runtime_ahead_of_index(report, "a")); + assert!(!runtime_ahead_of_index(report, "b")); + assert!(!runtime_ahead_of_index(report, "c")); + } + + #[test] + fn the_import_checks_run_where_their_comments_say() { + // A Windows checkout may use CRLF line endings. + let source = include_str!("e2e_prewarm.rs").replace("\r\n", "\n"); + let start = source.find("pub fn run(").expect("run exists"); + let end = source[start..].find("\n}\n").expect("run ends") + start; + let code: String = source[start..end] + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect::>() + .join("\n"); + let position = |needle: &str| { + code.find(needle) + .unwrap_or_else(|| panic!("`{needle}` is not called in run")) + }; + let repair = position("repair_torchvision_mismatch("); + let dangling = position("repair_dangling_active_runtime("); + let decision = position("let decision ="); + let engine = position("ensure_default_engine("); + let check = position("verify_runtime_imports("); + let reuse_return = position("if !runtime_changed(&decision)"); + assert!(repair < dangling && dangling < decision); + assert!(decision < engine && engine < check && check < reuse_return); + } } diff --git a/xtask/src/e2e_prewarm_vision.rs b/xtask/src/e2e_prewarm_vision.rs new file mode 100644 index 0000000000..6bab8b0f91 --- /dev/null +++ b/xtask/src/e2e_prewarm_vision.rs @@ -0,0 +1,1123 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The pre-warm's check that the active runtime can `import torchvision.ops`. +//! +//! Split out of [`crate::e2e_prewarm`], which calls [`repair_torchvision_mismatch`] +//! before it decides what to install and [`verify_runtime_imports`] after the +//! engine check. A persisted runtime can pass every freshness check and still hold +//! a torch and a torchvision that do not match; this finds that once, before the +//! suite, instead of in every vLLM serve scenario after its readiness timeout. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; + +use crate::e2e_prewarm::{StatusOk, default_engine, probe, rocm_command, runtime_ahead_of_index}; + +/// What importing torchvision's operators did in the active runtime's Python. +#[derive(Debug, PartialEq, Eq)] +enum VisionImport { + /// The runtime ships no torchvision, so there is nothing to check. + Absent, + /// `import torchvision.ops` succeeded. + Working, + /// It failed, or did not finish; the tail of stderr (or the deadline) says how. + Broken(String), +} + +/// What [`TORCHVISION_PROBE`] prints, and [`classify_vision_import`] reads, when +/// the runtime has no torchvision. A test ties the two together. +const ABSENT_MARKER: &str = "absent"; + +/// How long the probe may run before it is killed and reported as broken. An +/// import that does not finish is a runtime this pre-warm must not wait on. +const PROBE_TIMEOUT: Duration = Duration::from_mins(2); + +/// Run by the active runtime's Python. Prints [`ABSENT_MARKER`] and exits 0 when +/// the runtime has no torchvision, so such a runtime (a wheel runtime built +/// without it) is not mistaken for a broken one. `torchvision.ops` is the import +/// that fails when torchvision's compiled operators were built for a different +/// torch than the one installed next to it. +const TORCHVISION_PROBE: &str = "import importlib.util\n\ +if importlib.util.find_spec('torchvision') is None:\n print('absent')\n\ +else:\n import torchvision.ops\n"; + +/// Whether `text` carries torchvision's operator-mismatch error, such as +/// `operator torchvision::nms does not exist`. Both parts are required: this +/// decides whether a runtime is deleted, so an unrelated error that merely +/// mentions a `torchvision::` name must not match. +fn is_vision_mismatch(text: &str) -> bool { + text.contains("torchvision::") && text.contains("does not exist") +} + +/// Read what the probe did from its exit status and output. +fn classify_vision_import(success: bool, stdout: &str, stderr: &str) -> VisionImport { + if !success { + let lines: Vec<&str> = stderr + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + let mut tail: Vec<&str> = lines.iter().rev().take(3).rev().copied().collect(); + // The mismatch error is not always in the last lines (a chained + // traceback ends on the outer exception), and the callers read the + // signature from the tail, so keep the first line that carries it. + if !tail.iter().any(|line| is_vision_mismatch(line)) + && let Some(line) = lines.iter().find(|line| is_vision_mismatch(line)) + { + tail.insert(0, line); + } + let tail = if tail.is_empty() { + "the interpreter printed nothing to stderr".to_owned() + } else { + tail.join(" | ") + }; + return VisionImport::Broken(tail); + } + if stdout.trim() == ABSENT_MARKER { + VisionImport::Absent + } else { + VisionImport::Working + } +} + +/// How long to wait for a pipe's reader once the probe has ended. A grandchild +/// that inherited the pipe can hold it open after the probe is gone, and the +/// pre-warm must not wait on it. +const PIPE_GRACE: Duration = Duration::from_secs(2); + +/// Read a child's pipe to its end on its own thread, so a full pipe cannot stall +/// the child while the caller waits on its deadline. The bytes come back over a +/// channel so the caller can bound its wait; a reader still blocked on a pipe +/// something else holds is left running (detached) and ends when that closes. +fn drain(pipe: Option) -> std::sync::mpsc::Receiver> { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_end(&mut bytes); + } + let _ = sender.send(bytes); + }); + receiver +} + +/// Run [`TORCHVISION_PROBE`] with `python`, with `library_paths` (the runtime's +/// recorded ROCm library directories) on the loader path the way the CLI's own +/// probes put them, and kill it once `timeout` has passed. +fn probe_vision_import( + python: &Path, + library_paths: &[PathBuf], + timeout: Duration, +) -> std::io::Result { + let mut command = Command::new(python); + command + .args(["-c", TORCHVISION_PROBE]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if !library_paths.is_empty() { + // The recorded directories plus the amd-smi shim, ahead of whatever the + // job already exports: `runtime_loader_path_entries` is what a served + // process gets, so a probe that skips it judges a healthy runtime + // unloadable. + let mut entries = rocm_core::runtime_loader_path_entries(python, library_paths); + if let Some(existing) = std::env::var_os(rocm_core::RUNTIME_LIBRARY_PATH_ENV) { + entries.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(entries).map_err(std::io::Error::other)?; + command.env(rocm_core::RUNTIME_LIBRARY_PATH_ENV, joined); + } + // A script written an instant ago can be refused with `ETXTBSY` while another + // thread's fork still holds the write handle; that clears within + // milliseconds, so a few tries are made before giving up. + let mut attempts = 0; + let mut child = loop { + match command.spawn() { + Err(error) + if error.kind() == std::io::ErrorKind::ExecutableFileBusy && attempts < 20 => + { + attempts += 1; + std::thread::sleep(Duration::from_millis(50)); + } + spawned => break spawned?, + } + }; + let stdout = drain(child.stdout.take()); + let stderr = drain(child.stderr.take()); + let deadline = Instant::now() + timeout; + let status = loop { + if let Some(status) = child.try_wait()? { + break Some(status); + } + if Instant::now() >= deadline { + // Killing the probe closes its own ends of the pipes; a process it + // spawned may still hold them, which `PIPE_GRACE` below bounds. + let _ = child.kill(); + let _ = child.wait(); + break None; + } + std::thread::sleep(Duration::from_millis(50)); + }; + let read = |receiver: std::sync::mpsc::Receiver>| { + String::from_utf8_lossy(&receiver.recv_timeout(PIPE_GRACE).unwrap_or_default()).into_owned() + }; + let stdout = read(stdout); + let stderr = read(stderr); + let Some(status) = status else { + return Ok(VisionImport::Broken(format!( + "`import torchvision.ops` did not finish within {}s and was killed", + timeout.as_secs_f32() + ))); + }; + let verdict = classify_vision_import(status.success(), &stdout, &stderr); + Ok(match verdict { + // A crash during an ABI mismatch leaves no stderr, so say how it ended. + VisionImport::Broken(tail) => VisionImport::Broken(format!("{tail}; {status}")), + other => other, + }) +} + +/// The active runtime as the tree records it. +#[derive(Debug, PartialEq, Eq)] +struct ActiveRuntime { + key: String, + python: PathBuf, + /// `rocm_sdk.library_paths` from the runtime's manifest: the ROCm library + /// directories a served process gets on its loader path. Empty when the + /// manifest records none. + library_paths: Vec, +} + +/// The active runtime's key, Python and library directories, from the tree's +/// `active.json` and that runtime's registry manifest. +/// +/// `Ok(None)` only when there is nothing to check: the tree records no active +/// runtime, or the manifest records no interpreter (absent or null; the CLI +/// writes it as optional, so a renamed field cannot be told from that). A marker +/// or manifest that exists but cannot be read or parsed, or a marker without +/// `runtime_key` (which the CLI always writes), is an error. +fn active_runtime(prewarm_dir: &Path) -> Result> { + let runtimes = prewarm_dir.join("data").join("runtimes"); + let marker = runtimes.join("active.json"); + let active = match std::fs::read_to_string(&marker) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("failed to read {}", marker.display())); + } + }; + let active: serde_json::Value = serde_json::from_str(&active) + .with_context(|| format!("{} is not valid JSON", marker.display()))?; + let key = active + .get("runtime_key") + .and_then(serde_json::Value::as_str) + .with_context(|| format!("{} records no `runtime_key`", marker.display()))?; + let manifest_path = runtimes.join("registry").join(format!("{key}.json")); + let manifest = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("failed to read {}", manifest_path.display()))?; + let manifest: serde_json::Value = serde_json::from_str(&manifest) + .with_context(|| format!("{} is not valid JSON", manifest_path.display()))?; + let Some(python) = manifest + .get("python_executable") + .and_then(serde_json::Value::as_str) + else { + return Ok(None); + }; + let library_paths = manifest + .get("rocm_sdk") + .and_then(|sdk| sdk.get("library_paths")) + .and_then(serde_json::Value::as_array) + .map(|paths| { + paths + .iter() + .filter_map(serde_json::Value::as_str) + .map(PathBuf::from) + .collect() + }) + .unwrap_or_default(); + Ok(Some(ActiveRuntime { + key: key.to_owned(), + python: PathBuf::from(python), + library_paths, + })) +} + +/// What the torchvision check found in the active runtime. +#[derive(Debug, PartialEq, Eq)] +enum ImportCheck { + /// Nothing to check: no active runtime, or none with a recorded interpreter. + NoRuntime, + /// The runtime ships no torchvision. + Absent { key: String }, + /// `import torchvision.ops` worked. + Working { key: String }, + /// A shared library would not load and the manifest records no library + /// directories to have given it: the probe was not configured the way the + /// CLI's are, which says nothing about the runtime. + Unconfigured { key: String, tail: String }, + /// The import failed or hung. + Broken { key: String, tail: String }, +} + +/// Probe the active runtime's Python for `import torchvision.ops`. +fn check_runtime_imports(prewarm_dir: &Path, timeout: Duration) -> Result { + let Some(runtime) = active_runtime(prewarm_dir)? else { + return Ok(ImportCheck::NoRuntime); + }; + let key = runtime.key.clone(); + let verdict = probe_vision_import(&runtime.python, &runtime.library_paths, timeout) + .with_context(|| format!("failed to run the Python of runtime {key}"))?; + Ok(match verdict { + VisionImport::Absent => ImportCheck::Absent { key }, + VisionImport::Working => ImportCheck::Working { key }, + VisionImport::Broken(tail) + if runtime.library_paths.is_empty() + && tail.contains("cannot open shared object file") => + { + ImportCheck::Unconfigured { key, tail } + } + VisionImport::Broken(tail) => ImportCheck::Broken { key, tail }, + }) +} + +/// Whether serves on `engine` import torch, and so whether a runtime whose torch +/// and torchvision do not match is one they cannot use. +/// +/// The Lemonade engine runs its own llama.cpp backends and imports no torch +/// (`engines/lemonade/src/lib.rs` records `torch_version: None`), so a mismatch +/// in the runtime beside it does not fail its serves. Every other engine is +/// treated as needing torch: the cost of being wrong that way is a failed +/// pre-warm that names the cause, where the other way is a lane that waits out +/// every readiness timeout. +fn engine_imports_torch(engine: &str) -> bool { + engine != "lemonade" +} + +/// Remove the active runtime when its torch and torchvision do not match and the +/// engine about to serve needs torch, so the decision that follows sees a tree +/// without it and installs afresh. +/// +/// The same repair as `repair_poisoned_runtimes`: `rocm runtimes uninstall` drops +/// the registry entry, the active marker and the config pointers, and +/// `decide` on a registry with no runtime for the channel answers +/// `Decision::Install` (see the `no_managed_runtime_installs` test). Narrower on +/// purpose: only the operator-mismatch signature, because a hang, a library that +/// will not load or any other import error is not known to be cured by a +/// reinstall, and deleting a healthy runtime on a flaky probe would cost a +/// reinstall every run. Those reach [`verify_runtime_imports`] and fail there. +/// +/// Best effort until the uninstall itself: a tree whose engine or runtime state +/// cannot be read here is left for the fatal check after the engine install. +/// +/// Not done for a runtime the pre-warm must not replace: when the run is pinned +/// to an exact version, or the report says the runtime is `ahead_of_index` (see +/// `decide`: a reinstall could only roll it back). Those, and a report that +/// cannot be read, are declined: the runtime stays, this returns `true`, and +/// [`verify_runtime_imports`] fails after the engine check with the manual advice. +/// +/// Returns whether a mismatch was found and left in place. +pub(crate) fn repair_torchvision_mismatch( + rocm: &Path, + prewarm_dir: &Path, + pinned: bool, +) -> Result { + repair_torchvision_mismatch_within(rocm, prewarm_dir, PROBE_TIMEOUT, pinned) +} + +fn repair_torchvision_mismatch_within( + rocm: &Path, + prewarm_dir: &Path, + timeout: Duration, + pinned: bool, +) -> Result { + let Ok(engine) = default_engine(rocm, prewarm_dir) else { + return Ok(false); + }; + if !engine_imports_torch(&engine) { + return Ok(false); + } + let Ok(ImportCheck::Broken { key, tail }) = check_runtime_imports(prewarm_dir, timeout) else { + return Ok(false); + }; + if !is_vision_mismatch(&tail) { + return Ok(false); + } + let declined = if pinned { + Some("this pre-warm is pinned to an exact version".to_owned()) + } else { + match probe(rocm, prewarm_dir) { + Ok(report) if runtime_ahead_of_index(&report, &key) => Some( + "it is ahead of the channel index, so a reinstall would roll it back".to_owned(), + ), + Ok(_) => None, + Err(error) => Some(format!( + "`rocm update` could not say whether it is ahead of the index ({error:#})" + )), + } + }; + if let Some(why) = declined { + println!( + "::warning::pre-warm: runtime {key} cannot `import torchvision.ops` ({tail}), so its \ + torch and torchvision do not match; it is not removed because {why}." + ); + return Ok(true); + } + println!( + "::warning::pre-warm: removing the shared runtime {key} — it cannot `import \ + torchvision.ops` ({tail}), so its torch and torchvision do not match. A reinstall follows." + ); + rocm_command(rocm, prewarm_dir) + .args(["runtimes", "uninstall", &key, "--yes"]) + .status_ok("rocm runtimes uninstall")?; + Ok(false) +} + +/// Fail the pre-warm when the active runtime cannot import torchvision's operators +/// and the engine serving on it needs them. +/// +/// A persisted runtime can pass every freshness check and still be broken: when +/// its torch and torchvision come from different installs, torchvision's compiled +/// operators no longer match torch and `import torchvision` dies with `operator +/// torchvision::nms does not exist`. Nothing before the suite imports it, so each +/// vLLM serve scenario would find out for itself after its readiness timeout. +/// This finds it once, before the suite starts. +/// +/// Fatal where `engine` imports torch, like `ensure_default_engine`: serving +/// against a runtime already known to be broken fails later and names none of +/// this. For an engine that does not, the same finding is a warning. The probe +/// targets the runtime's own Python because that is where the engine runs: the +/// vLLM install lays its packages, torchvision included, into the managed +/// runtime's interpreter (`install_vllm_with_uv(&managed.python_executable, ..)` in +/// `engines/vllm/src/install.rs`) and creates no venv of its own. +pub(crate) fn verify_runtime_imports( + rocm: &Path, + prewarm_dir: &Path, + installed_this_run: bool, + engine: &str, + repair_declined: bool, +) -> Result<()> { + match check_runtime_imports(prewarm_dir, PROBE_TIMEOUT)? { + ImportCheck::NoRuntime => println!( + "pre-warm: no active runtime interpreter recorded; skipping the torchvision check" + ), + ImportCheck::Absent { key } => { + println!("pre-warm: runtime {key} has no torchvision; nothing to check"); + } + ImportCheck::Working { key } => { + println!("pre-warm: runtime {key} imports torchvision.ops"); + } + // Last resort: reached only when the manifest records no library + // directories, so nothing could be put on the loader path. + ImportCheck::Unconfigured { key, tail } => println!( + "::warning::pre-warm: runtime {key} could not load a shared library for the \ + torchvision check and records no library directories to supply ({tail}); skipping it" + ), + ImportCheck::Broken { key, tail } => { + let mismatch = if is_vision_mismatch(&tail) { + " Its torch and torchvision do not match." + } else { + "" + }; + if !engine_imports_torch(engine) { + println!( + "::warning::pre-warm: runtime {key} cannot `import torchvision.ops` \ + ({tail}).{mismatch} The {engine} engine does not import torch, so this \ + does not fail the pre-warm." + ); + return Ok(()); + } + let advice = if installed_this_run { + "This run installed the runtime, so the installer produced the broken pair and \ + a reinstall would reproduce it; fix the install." + .to_owned() + } else if mismatch.is_empty() || repair_declined { + format!( + "To reinstall it, set ROCM_CLI_CONFIG_DIR={dir}/config, \ + ROCM_CLI_DATA_DIR={dir}/data and ROCM_CLI_CACHE_DIR={dir}/cache, run \ + `{rocm} runtimes uninstall {key} --yes`, and run the pre-warm again.", + dir = prewarm_dir.display(), + rocm = rocm.display() + ) + } else { + // `repair_torchvision_mismatch` removes this runtime at the start + // of the next pre-warm, once it is the active one. + "Run the pre-warm again: it removes a runtime whose torch and torchvision do \ + not match and installs afresh." + .to_owned() + }; + bail!( + "runtime {key} cannot `import torchvision.ops` ({tail}).{mismatch} Serves that \ + import torch would fail. {advice}" + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Tests that write a script and then run it hold this lock for their whole + /// body. Writing an executable and exec'ing it fails with `ETXTBSY` when + /// another thread forks while the write handle is still open, and these + /// tests fork constantly. + #[cfg(unix)] + fn serial() -> std::sync::MutexGuard<'static, ()> { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + LOCK.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[test] + fn vision_probe_is_read_from_exit_status_and_output() { + assert_eq!( + classify_vision_import(true, "", ""), + VisionImport::Working, + "a clean import prints nothing" + ); + assert_eq!( + classify_vision_import(true, "absent\n", ""), + VisionImport::Absent + ); + // The failure that cost the MI300X lane its green runs: only the tail of + // the traceback is kept, and the cause is its last line. + let stderr = "Traceback (most recent call last):\n File \"x.py\", line 1\n a\n b\n\ + RuntimeError: operator torchvision::nms does not exist\n"; + match classify_vision_import(false, "", stderr) { + VisionImport::Broken(tail) => { + assert!(tail.ends_with("RuntimeError: operator torchvision::nms does not exist")); + assert_eq!(tail.matches(" | ").count(), 2, "three lines kept: {tail}"); + } + other => panic!("expected Broken, got {other:?}"), + } + // A failure that prints "absent" first is still a failure. + assert!(matches!( + classify_vision_import(false, "absent\n", "boom"), + VisionImport::Broken(_) + )); + } + + /// A tree whose active runtime records `python`, laid out as the CLI does. + fn tree_with_python(python: &Path) -> tempfile::TempDir { + tree_with_runtime(python, &[]) + } + + /// [`tree_with_python`] whose manifest also records `library_paths`. + fn tree_with_runtime(python: &Path, library_paths: &[&Path]) -> tempfile::TempDir { + let tree = tempfile::tempdir().expect("tempdir"); + let runtimes = tree.path().join("data").join("runtimes"); + std::fs::create_dir_all(runtimes.join("registry")).expect("registry dir"); + std::fs::write( + runtimes.join("active.json"), + r#"{"runtime_key": "release-wheel-test"}"#, + ) + .expect("active.json"); + let manifest = if library_paths.is_empty() { + serde_json::json!({ "python_executable": python }) + } else { + serde_json::json!({ + "python_executable": python, + "rocm_sdk": { "library_paths": library_paths }, + }) + }; + std::fs::write( + runtimes.join("registry").join("release-wheel-test.json"), + manifest.to_string(), + ) + .expect("manifest"); + tree + } + + #[test] + fn active_runtime_follows_the_active_marker_to_its_manifest() { + let tree = tree_with_python(Path::new("/runtime/bin/python")); + assert_eq!( + active_runtime(tree.path()).expect("reads"), + Some(ActiveRuntime { + key: "release-wheel-test".to_owned(), + python: PathBuf::from("/runtime/bin/python"), + library_paths: Vec::new(), + }) + ); + // The library directories the CLI records under `rocm_sdk` are read too. + let with_libs = tree_with_runtime( + Path::new("/runtime/bin/python"), + &[Path::new("/sdk/lib"), Path::new("/sdk/bin")], + ); + assert_eq!( + active_runtime(with_libs.path()) + .expect("reads") + .expect("an active runtime") + .library_paths, + vec![PathBuf::from("/sdk/lib"), PathBuf::from("/sdk/bin")] + ); + // No marker, nothing to check. + let empty = tempfile::tempdir().expect("tempdir"); + assert_eq!(active_runtime(empty.path()).expect("reads"), None); + } + + #[test] + fn unreadable_runtime_state_is_an_error_not_a_skip() { + let tree = tree_with_python(Path::new("/runtime/bin/python")); + let runtimes = tree.path().join("data").join("runtimes"); + // A marker that exists but is not JSON. + std::fs::write(runtimes.join("active.json"), "not json").expect("corrupt marker"); + assert!(active_runtime(tree.path()).is_err()); + assert!( + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false).is_err() + ); + // A marker naming a runtime whose manifest is gone. + std::fs::write( + runtimes.join("active.json"), + r#"{"runtime_key": "no-such-runtime"}"#, + ) + .expect("marker"); + assert!(active_runtime(tree.path()).is_err()); + // A marker that cannot be read at all (here, a directory) is not "absent". + std::fs::remove_file(runtimes.join("active.json")).expect("remove marker"); + std::fs::create_dir(runtimes.join("active.json")).expect("directory marker"); + assert!(active_runtime(tree.path()).is_err()); + std::fs::remove_dir(runtimes.join("active.json")).expect("remove directory"); + // A marker without a runtime key is a shape change, not a skip. + std::fs::write(runtimes.join("active.json"), "{}").expect("marker"); + assert!(active_runtime(tree.path()).is_err()); + // A manifest that exists but is not JSON. + std::fs::write( + runtimes.join("active.json"), + r#"{"runtime_key": "release-wheel-test"}"#, + ) + .expect("marker"); + std::fs::write( + runtimes.join("registry").join("release-wheel-test.json"), + "not json", + ) + .expect("corrupt manifest"); + assert!(active_runtime(tree.path()).is_err()); + // A manifest that records no interpreter leaves nothing to check. + std::fs::write( + runtimes.join("registry").join("release-wheel-test.json"), + r#"{"python_executable": null}"#, + ) + .expect("manifest"); + std::fs::write( + runtimes.join("active.json"), + r#"{"runtime_key": "release-wheel-test"}"#, + ) + .expect("marker"); + assert_eq!(active_runtime(tree.path()).expect("reads"), None); + } + + #[test] + fn the_probe_prints_the_marker_the_classifier_reads() { + assert!(TORCHVISION_PROBE.contains(&format!("print('{ABSENT_MARKER}')"))); + assert_eq!( + classify_vision_import(true, &format!("{ABSENT_MARKER}\n"), ""), + VisionImport::Absent + ); + assert!( + matches!( + classify_vision_import(false, "", "\n\n"), + VisionImport::Broken(ref tail) if tail.contains("nothing to stderr") + ), + "an empty stderr still says something" + ); + } + + /// A stand-in interpreter: a script that ignores `-c` and behaves as told. + #[cfg(unix)] + fn fake_python(dir: &Path, body: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + let path = dir.join("python"); + std::fs::write(&path, format!("#!/bin/sh\n{body}\n")).expect("script"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + path + } + + #[cfg(unix)] + #[test] + fn the_probe_is_passed_to_the_interpreter_with_dash_c() { + let _serial = serial(); + // Succeeds only when its first argument is `-c`, as the probe passes it. + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python( + scripts.path(), + "[ \"$1\" = -c ] || { echo \"unexpected arguments: $*\" >&2; exit 7; }", + ); + assert_eq!( + probe_vision_import(&python, &[], PROBE_TIMEOUT).expect("runs"), + VisionImport::Working + ); + } + + #[cfg(unix)] + #[test] + fn the_advice_depends_on_whether_this_run_installed_the_runtime() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python(scripts.path(), "echo 'ImportError: boom' >&2; exit 1"); + let tree = tree_with_python(&python); + let reused = format!( + "{:#}", + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false) + .expect_err("fails") + ); + assert!( + reused.contains("runtimes uninstall release-wheel-test --yes"), + "{reused}" + ); + assert!( + !reused.contains("do not match"), + "no torchvision:: in the tail: {reused}" + ); + let fresh = format!( + "{:#}", + verify_runtime_imports(Path::new("rocm"), tree.path(), true, "vllm", false) + .expect_err("fails") + ); + assert!(fresh.contains("This run installed the runtime"), "{fresh}"); + assert!(!fresh.contains("runtimes uninstall"), "{fresh}"); + } + + #[cfg(unix)] + #[test] + fn the_runtimes_library_directories_reach_the_probe() { + let _serial = serial(); + // Exits 0 only when its loader path starts with the recorded directory. + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python( + scripts.path(), + "case \"$LD_LIBRARY_PATH\" in /sdk/lib:*|/sdk/lib) exit 0 ;; \ + *) echo \"loader path was: $LD_LIBRARY_PATH\" >&2; exit 3 ;; esac", + ); + let with_libs = tree_with_runtime(&python, &[Path::new("/sdk/lib")]); + assert!( + verify_runtime_imports(Path::new("rocm"), with_libs.path(), false, "vllm", false) + .is_ok() + ); + } + + #[cfg(unix)] + #[test] + fn a_library_that_will_not_load_fails_when_directories_were_supplied() { + let _serial = serial(); + // The runtime's own directories were on the path and the load still + // failed: that is a broken runtime, not an unconfigured probe. + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python( + scripts.path(), + "echo 'libroctx64.so.4: cannot open shared object file' >&2; exit 1", + ); + let tree = tree_with_runtime(&python, &[Path::new("/sdk/lib")]); + let error = verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false) + .expect_err("must fail"); + assert!( + format!("{error:#}").contains("libroctx64.so.4"), + "{error:#}" + ); + } + + #[cfg(unix)] + #[test] + fn a_library_that_will_not_load_is_skipped_only_when_nothing_could_be_supplied() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python( + scripts.path(), + "echo 'libroctx64.so.4: cannot open shared object file' >&2; exit 1", + ); + let tree = tree_with_python(&python); + assert!(matches!( + check_runtime_imports(tree.path(), PROBE_TIMEOUT).expect("runs"), + ImportCheck::Unconfigured { .. } + )); + assert!( + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false).is_ok() + ); + } + + #[cfg(unix)] + #[test] + fn a_probe_that_never_finishes_is_killed_and_reported() { + let _serial = serial(); + // `exec` so the killed process is the sleeper itself and holds the pipes. + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python(scripts.path(), "exec sleep 60"); + let started = Instant::now(); + let verdict = probe_vision_import(&python, &[], Duration::from_millis(300)).expect("runs"); + assert!( + started.elapsed() < Duration::from_secs(30), + "the deadline held" + ); + match verdict { + VisionImport::Broken(tail) => assert!(tail.contains("did not finish"), "{tail}"), + other => panic!("expected Broken, got {other:?}"), + } + } + + #[cfg(unix)] + const MISMATCH: &str = + "echo 'RuntimeError: operator torchvision::nms does not exist' >&2; exit 1"; + + #[cfg(unix)] + #[test] + fn a_runtime_the_engine_does_not_need_torch_for_only_warns() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let tree = tree_with_python(&fake_python(scripts.path(), MISMATCH)); + assert!( + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "lemonade", false) + .is_ok() + ); + assert!( + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false).is_err() + ); + } + + /// A stand-in `rocm`: `engines list` marks `engine` as the default, every call + /// is appended to `calls`, and `runtimes uninstall ` removes that + /// runtime's registry entry and the active marker from the tree the call is + /// scoped to, as the CLI does. `update` prints the `update_report` file in `dir` + /// and fails when `update_fails` exists. + #[cfg(unix)] + fn fake_rocm(dir: &Path, engine: &str) -> (PathBuf, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let calls = dir.join("calls"); + let path = dir.join("rocm"); + std::fs::write( + &path, + format!( + "#!/bin/sh\necho \"$*\" >> {calls}\n\ + [ \"$1 $2\" = \"engines list\" ] && echo '* {engine} default'\n\ + if [ \"$1\" = update ]; then\n\ + [ -e {dir}/update_fails ] && exit 1\n\ + cat {dir}/update_report 2>/dev/null\nexit 0\nfi\n\ + if [ \"$1 $2\" = \"runtimes uninstall\" ]; then\n\ + rm -f \"$ROCM_CLI_DATA_DIR/runtimes/registry/$3.json\" \ + \"$ROCM_CLI_DATA_DIR/runtimes/active.json\"\nfi\nexit 0\n", + calls = calls.display(), + dir = dir.display() + ), + ) + .expect("script"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + (path, calls) + } + + #[cfg(unix)] + fn uninstalls(calls: &Path) -> String { + std::fs::read_to_string(calls) + .unwrap_or_default() + .lines() + .filter(|line| line.starts_with("runtimes uninstall")) + .collect::>() + .join("\n") + } + + #[cfg(unix)] + #[test] + fn a_torch_and_torchvision_mismatch_is_uninstalled_so_the_next_decision_installs() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let (rocm, calls) = fake_rocm(scripts.path(), "vllm"); + let tree = tree_with_python(&fake_python(scripts.path(), MISMATCH)); + let manifest = tree + .path() + .join("data/runtimes/registry/release-wheel-test.json"); + assert!(manifest.exists()); + repair_torchvision_mismatch(&rocm, tree.path(), false).expect("repairs"); + assert_eq!( + uninstalls(&calls), + "runtimes uninstall release-wheel-test --yes" + ); + // Red if the uninstall is not requested for this runtime: the stand-in + // removes the registry entry only for the key it is given. That the + // emptied registry then installs afresh is `decide`'s own behavior, + // pinned by `no_managed_runtime_installs` in `e2e_prewarm.rs`. + assert!(!manifest.exists()); + assert_eq!(active_runtime(tree.path()).expect("reads"), None); + } + + #[cfg(unix)] + #[test] + fn a_runtime_the_pre_warm_must_not_replace_is_not_uninstalled() { + let _serial = serial(); + let line = + |status: &str| format!("runtime release-wheel-test channel=release status={status}\n"); + // (pinned, `rocm update` report, `rocm update` fails, why) + let cases: [(bool, String, bool, &str); 3] = [ + (true, line("update_available"), false, "a pinned run"), + ( + false, + line("ahead_of_index"), + false, + "a runtime ahead of the index", + ), + (false, String::new(), true, "a report that cannot be read"), + ]; + for (pinned, report, fails, why) in cases { + let scripts = tempfile::tempdir().expect("tempdir"); + let (rocm, calls) = fake_rocm(scripts.path(), "vllm"); + std::fs::write(scripts.path().join("update_report"), report).expect("report"); + if fails { + std::fs::write(scripts.path().join("update_fails"), "").expect("marker"); + } + let tree = tree_with_python(&fake_python(scripts.path(), MISMATCH)); + let declined = repair_torchvision_mismatch(&rocm, tree.path(), pinned).expect("runs"); + assert!(declined, "{why}: the mismatch is reported as left in place"); + assert_eq!(uninstalls(&calls), "", "{why}"); + } + // The same mismatch, with the index ahead of the runtime, is removed. + let scripts = tempfile::tempdir().expect("tempdir"); + let (rocm, calls) = fake_rocm(scripts.path(), "vllm"); + std::fs::write( + scripts.path().join("update_report"), + line("update_available"), + ) + .expect("report"); + let tree = tree_with_python(&fake_python(scripts.path(), MISMATCH)); + assert!(!repair_torchvision_mismatch(&rocm, tree.path(), false).expect("runs")); + assert_eq!( + uninstalls(&calls), + "runtimes uninstall release-wheel-test --yes" + ); + } + + #[cfg(unix)] + #[test] + fn the_advice_for_a_mismatch_depends_on_whether_the_repair_was_declined() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let tree = tree_with_python(&fake_python(scripts.path(), MISMATCH)); + let message = |declined| { + format!( + "{:#}", + verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", declined) + .expect_err("fails") + ) + }; + assert!(message(false).contains("Run the pre-warm again")); + let declined = message(true); + assert!( + declined.contains("runtimes uninstall release-wheel-test --yes") + && !declined.contains("Run the pre-warm again"), + "{declined}" + ); + } + + #[cfg(unix)] + #[test] + fn only_a_real_mismatch_on_a_torch_engine_is_uninstalled() { + let _serial = serial(); + // Another import error, a hang, a working runtime, a runtime without + // torchvision and a Lemonade lane are all left alone. + let cases: [(&str, &str, &str); 6] = [ + ( + "vllm", + "echo 'ImportError: boom' >&2; exit 1", + "another import error", + ), + ( + "vllm", + "echo 'ImportError: torchvision::nms failed to load' >&2; exit 1", + "an error that names a torchvision operator without saying it is missing", + ), + ("vllm", "exit 0", "a working runtime"), + ("vllm", "echo absent", "no torchvision"), + ("lemonade", MISMATCH, "an engine that does not import torch"), + // `exec` so the killed process is the sleeper itself. + ("vllm", "exec sleep 60", "a probe that never finishes"), + ]; + for (engine, body, why) in cases { + let scripts = tempfile::tempdir().expect("tempdir"); + let (rocm, calls) = fake_rocm(scripts.path(), engine); + let tree = tree_with_python(&fake_python(scripts.path(), body)); + repair_torchvision_mismatch_within( + &rocm, + tree.path(), + Duration::from_millis(300), + false, + ) + .expect("runs"); + assert_eq!(uninstalls(&calls), "", "{why}"); + } + } + + #[cfg(unix)] + #[test] + fn a_grandchild_holding_the_pipes_cannot_stall_the_probe() { + let _serial = serial(); + // The probe starts a background process that inherits its pipes, then + // becomes a sleeper. Killing the probe leaves the grandchild holding + // both pipes open, so the readers never see the end of the stream. + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python(scripts.path(), "sleep 60 &\nexec sleep 60"); + let started = Instant::now(); + let verdict = probe_vision_import(&python, &[], Duration::from_millis(300)).expect("runs"); + assert!( + started.elapsed() < Duration::from_secs(30), + "the wait on the readers is bounded: {:?}", + started.elapsed() + ); + assert!(matches!(verdict, VisionImport::Broken(ref t) if t.contains("did not finish"))); + } + + #[test] + fn a_mismatch_error_above_the_last_lines_is_still_recognised() { + let stderr = "RuntimeError: operator torchvision::nms does not exist\n\ + During handling of the above exception, another exception occurred:\n\ + Traceback (most recent call last):\n File \"x.py\", line 1\n\ + ImportError: could not import torchvision.ops\n"; + match classify_vision_import(false, "", stderr) { + VisionImport::Broken(tail) => { + assert!(is_vision_mismatch(&tail), "{tail}"); + assert!(tail.ends_with("could not import torchvision.ops"), "{tail}"); + } + other => panic!("expected Broken, got {other:?}"), + } + } + + #[cfg(unix)] + #[test] + fn a_runtime_that_cannot_import_torchvision_ops_fails_the_pre_warm() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let python = fake_python( + scripts.path(), + "echo 'RuntimeError: operator torchvision::nms does not exist' >&2; exit 1", + ); + let tree = tree_with_python(&python); + let error = verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false) + .expect_err("must fail"); + let message = format!("{error:#}"); + assert!( + message.starts_with("runtime release-wheel-test cannot"), + "{message}" + ); + assert!( + message.contains(" Its torch and torchvision do not match."), + "{message}" + ); + // A mismatch is what `repair_torchvision_mismatch` removes at the start of + // the next pre-warm (`a_torch_and_torchvision_mismatch_is_uninstalled_…`), + // so the advice is to run it again. + assert!( + message.contains("Run the pre-warm again: it removes a runtime whose torch"), + "{message}" + ); + assert!(message.contains("exit status: 1"), "{message}"); + assert!( + message.contains("torchvision::nms does not exist"), + "{message}" + ); + } + + #[cfg(unix)] + #[test] + fn working_and_torchvision_free_runtimes_pass_the_pre_warm() { + let _serial = serial(); + let scripts = tempfile::tempdir().expect("tempdir"); + let working = tree_with_python(&fake_python(scripts.path(), "exit 0")); + assert!( + verify_runtime_imports(Path::new("rocm"), working.path(), false, "vllm", false).is_ok() + ); + + let scripts = tempfile::tempdir().expect("tempdir"); + let absent = tree_with_python(&fake_python(scripts.path(), "echo absent")); + assert!( + verify_runtime_imports(Path::new("rocm"), absent.path(), false, "vllm", false).is_ok() + ); + + // No active runtime recorded: skipped, not failed. + let empty = tempfile::tempdir().expect("tempdir"); + assert!( + verify_runtime_imports(Path::new("rocm"), empty.path(), false, "vllm", false).is_ok() + ); + } + + #[test] + fn a_recorded_interpreter_that_cannot_start_fails_the_pre_warm() { + // The manifest names a Python that is not there: the runtime is not + // usable, which is the same verdict as one that cannot import. + let tree = tree_with_python(Path::new("/nonexistent/runtime/bin/python")); + let error = verify_runtime_imports(Path::new("rocm"), tree.path(), false, "vllm", false) + .expect_err("must fail"); + assert!( + format!("{error:#}").contains("failed to run the Python of runtime release-wheel-test") + ); + } + + /// Runs the real probe with the host's `python3` against a throwaway + /// `torchvision` package, so the probe text itself is exercised and not only + /// the classifier. Skipped where no `python3` is on the PATH. + #[cfg(unix)] + #[test] + fn the_probe_imports_torchvision_ops_in_a_real_interpreter() { + let _serial = serial(); + let python = Path::new("python3"); + let usable = Command::new(python) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()); + // Skipped, and said so, on an image without `python3`: the probe is run + // by the runtime's own interpreter in the pre-warm, not by this one. + if !usable { + eprintln!("skipping: no working python3 on PATH"); + return; + } + let write_package = |ops: &str| { + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("torchvision"); + std::fs::create_dir_all(&package).expect("package dir"); + std::fs::write(package.join("__init__.py"), "").expect("__init__"); + std::fs::write(package.join("ops.py"), ops).expect("ops"); + dir + }; + let classify = |output: std::process::Output| { + classify_vision_import( + output.status.success(), + &String::from_utf8_lossy(&output.stdout), + &String::from_utf8_lossy(&output.stderr), + ) + }; + let run = |path: &Path| { + // The package goes on sys.path by hand so the probe text runs as is. + let script = format!( + "import sys; sys.path.insert(0, {:?})\n{TORCHVISION_PROBE}", + path.to_string_lossy() + ); + classify( + Command::new(python) + .args(["-c", &script]) + .output() + .expect("python3 runs"), + ) + }; + + let working = write_package(""); + assert_eq!(run(working.path()), VisionImport::Working); + + let broken = + write_package("raise RuntimeError('operator torchvision::nms does not exist')"); + match run(broken.path()) { + VisionImport::Broken(tail) => { + assert!(tail.contains("torchvision::nms does not exist"), "{tail}"); + } + other => panic!("expected Broken, got {other:?}"), + } + + // `-I -S`: no site-packages and no current directory, so no torchvision + // can be found and the probe takes its `find_spec` branch. + let absent = classify( + Command::new(python) + .args(["-I", "-S", "-c", TORCHVISION_PROBE]) + .output() + .expect("python3 runs"), + ); + assert_eq!(absent, VisionImport::Absent); + } +} diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs new file mode 100644 index 0000000000..849d3a7152 --- /dev/null +++ b/xtask/src/e2e_queue_import.rs @@ -0,0 +1,942 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bring a finished run of the shared GPU queue into the nightly's consolidated +//! E2E report, so the support matrix and the release-readiness verdict derived +//! from it keep covering the GPUs the queue tests. +//! +//! `nightly.yml`'s `e2e-report-nightly` merges one `report.json` per +//! `e2e-*-report` artifact. The queue publishes a different layout: one +//! directory per lane in its `queue-run-results` artifact, holding +//! `/results/{report.json,platform.json,commands.jsonl,...}` as +//! `scripts/run-e2e-lane.sh` writes them (docs/nightly-on-the-queue.md). This +//! turns the one into the other: +//! +//! * every lane's reports are copied to `/e2e-queue-[-advisory]--report/`, +//! named so `e2e_report` labels them by lane and gate. Only the three files the +//! report reads are copied; `lane.log`, the JUnit and HTML output and the +//! harness diagnostics stay where they are. +//! * the report's `platform_slug` is replaced by the lane's name and channel. The +//! slug is what the report keys a grid column by, and queue lanes of different +//! models report one slug (an MI325X lane probes as `mi300x`, an MI355X as +//! `mi350p`), which would fold them, and the self-hosted MI300X lane, into one +//! column. The channel is there because a lane's manifest names the channel of +//! the runtime it found, which need not be the one the lane was told to run. +//! * which lanes exist and which of them the queue blocks on is read from the +//! run's lane matrix in its job log, the only place either is published. A lane +//! the queue marks advisory is labelled so, and `release-readiness` leaves it +//! out of its verdict. Anything this cannot read counts as blocking. +//! * a lane that reported nothing, or that did not finish cleanly and lacks a +//! channel, gets an empty report for it, which the report shows as `EMPTY` and +//! `release-readiness` counts as a failure for a blocking lane: a lane without +//! a result is not a pass. +//! * a short status table of every lane is written for the top of the page. +//! +//! Nothing is imported unless the caller found a run: with `--absent ` +//! only the status section is written, saying so, and the nightly report goes on +//! without queue lanes. The repository is public, so everything written is either +//! copied from the queue's own scanned files or built from fields this checks +//! against a strict pattern first. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; + +use anyhow::{Context, Result, bail}; +use regex::Regex; + +/// The channels `scripts/run-e2e-lane.sh` runs, in the order it runs them. +const CHANNELS: [&str; 2] = ["release", "nightly"]; + +/// The files of a lane's `/results` the consolidated report reads. +const REPORT_FILES: [&str; 3] = ["report.json", "platform.json", "commands.jsonl"]; + +#[derive(clap::Args)] +pub struct Args { + /// Directory the queue run's `queue-run-results` artifact was downloaded + /// into: one subdirectory per lane. + #[arg(long, requires = "run_url", conflicts_with = "absent")] + queue_dir: Option, + /// The queue run's job log, which prints the branch the run tested and its + /// lane matrix. Without it, or when it names a branch other than `main`, + /// nothing is imported. + #[arg(long, requires = "queue_dir")] + log: Option, + /// The queue run's URL, linked from the status section. + #[arg(long)] + run_url: Option, + /// When the queue run started, as the API reports it. + #[arg(long, requires = "queue_dir")] + run_started: Option, + /// Import nothing and say why: no queue run was found, or its results could + /// not be fetched. + #[arg( + long, + conflicts_with = "queue_dir", + required_unless_present = "queue_dir" + )] + absent: Option, + /// Where the per-lane report directories go: the directory + /// `xtask e2e-report` reads. + #[arg(long)] + artifacts_dir: PathBuf, + /// Where to write the status section, for the top of the support matrix. + #[arg(long)] + section_out: PathBuf, +} + +pub fn run(args: &Args) -> Result<()> { + let section = match (&args.queue_dir, &args.absent) { + (Some(queue_dir), _) => { + let log = args.log.as_ref().and_then(|path| { + std::fs::read_to_string(path) + .map_err(|e| eprintln!("warning: queue job log {}: {e}", path.display())) + .ok() + }); + let log = parse_log(log.as_deref()); + if log.branch.as_deref() == Some("main") { + // A log that yielded no lanes is as good as none for the lanes' + // state: every lane is then read as blocking. + let have_matrix = !log.lanes.is_empty(); + let lanes_plan = plan(&log, scan(queue_dir)?, have_matrix); + write_reports(&lanes_plan, queue_dir, &args.artifacts_dir)?; + render_section( + &lanes_plan, + args.run_url.as_deref(), + args.run_started.as_deref(), + have_matrix, + ) + } else { + // The run is of the workflow on `main`, but may have been asked + // to test another branch, which is no night's result for `main`. + // The log is the only place that says which, so without it + // nothing is imported. + render_absent( + "the newest queue run did not test main, or its job log could not be read \ + to tell", + ) + } + } + (None, Some(reason)) => render_absent(reason), + (None, None) => bail!("one of --queue-dir and --absent is required"), + }; + if let Some(parent) = args.section_out.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + std::fs::write(&args.section_out, section) + .with_context(|| format!("writing {}", args.section_out.display()))?; + Ok(()) +} + +/// What the queue run's job log says about the run. +#[derive(Debug, Default, PartialEq, Eq)] +struct QueueLog { + /// The branch the run tested and the commit every lane tested, from the + /// report's first line. The branch is not the one the workflow ran on: a run + /// started from `main` can be asked to test another branch. + branch: Option, + commit: Option, + lanes: Vec, +} + +/// One row of the lane matrix the queue prints. +#[derive(Debug, PartialEq, Eq)] +struct MatrixRow { + lane: String, + /// The lane's phase (`Done`, `Failed`, ...), when the row had the usual shape. + phase: Option, + /// The lane command's exit status, or `-` when it never ran. + exit: Option, + /// Whether the queue lets this lane fail the run. Anything but `advisory` is + /// read as blocking. + blocking: bool, +} + +impl MatrixRow { + /// A lane the queue reports as finished with exit status zero. + fn is_clean(&self) -> bool { + self.phase.as_deref() == Some("Done") && self.exit.as_deref() == Some("0") + } +} + +/// A lane name is a path segment and a word in an artifact name: lowercase +/// letters, digits and dashes only. +/// +/// A name ending in a word the artifact names use for something else +/// (`-advisory`, `-release`, `-nightly`) is refused too: it would be read back +/// as a gate or a channel. +fn is_lane_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 40 + && !name.starts_with('-') + && !["-advisory", "-release", "-nightly"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Read the lane matrix and the tested commit out of the run's job log. +/// +/// The publisher prints its report with a `| ` prefix on every line, which the +/// runner puts a timestamp in front of. Only the fields that match a strict +/// pattern are kept; a line that does not is ignored, so a log that is not what +/// was expected yields no lanes rather than text to publish. +fn parse_log(log: Option<&str>) -> QueueLog { + static TESTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern") + }); + let mut out = QueueLog::default(); + let Some(log) = log else { return out }; + let tested = &*TESTED; + let mut in_matrix = false; + let mut unnamed = 0; + for line in log.lines() { + let line = line.trim_start_matches('\u{feff}'); + // ` | `: the part after the first `| `. + let Some((_, body)) = line.split_once("| ") else { + // A blank report line is `|` alone; either way the matrix is over. + if line.trim_end().ends_with('|') { + in_matrix = false; + } + continue; + }; + let body = body.trim_end(); + if let Some(caps) = tested.captures(body) { + out.branch = Some(caps[1].to_owned()); + out.commit = Some(caps[2].to_owned()); + } + let tokens: Vec<&str> = body.split_whitespace().collect(); + if tokens.first() == Some(&"LANE") && tokens.last() == Some(&"GATE") { + in_matrix = true; + continue; + } + if !in_matrix { + continue; + } + if body.is_empty() || tokens.first() == Some(&"SUMMARY:") { + in_matrix = false; + continue; + } + let (Some(lane), Some(gate)) = (tokens.first(), tokens.last()) else { + continue; + }; + if out.lanes.iter().any(|row| row.lane == *lane) { + continue; + } + // A row whose lane cannot be named safely still counts: as a lane that + // reported nothing, and blocking, since its gate cannot be trusted either. + if !is_lane_name(lane) { + unnamed += 1; + out.lanes.push(MatrixRow { + lane: format!("unnamed-row-{unnamed}"), + phase: None, + exit: None, + blocking: true, + }); + continue; + } + // LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE + let shaped = tokens.len() == 7; + let phase = shaped + .then(|| tokens[2]) + .filter(|p| p.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_owned); + let exit = shaped + .then(|| tokens[3]) + .filter(|e| *e == "-" || (e.len() <= 4 && e.bytes().all(|b| b.is_ascii_digit()))) + .map(str::to_owned); + out.lanes.push(MatrixRow { + lane: (*lane).to_owned(), + phase, + exit, + blocking: *gate != "advisory", + }); + } + out +} + +/// The channels with a `results/report.json` under each lane directory of the +/// downloaded artifact. A directory (or the final file) that is a symlink is not +/// followed. A lane directory with a name that is not a lane name is kept as an +/// `unnamed-dir-N` lane with no reports (a matrix row is `unnamed-row-N`: the two +/// cannot be told to be the same lane, so they are never merged), so that it shows as a lane without a result +/// instead of dropping out unseen. +fn scan(queue_dir: &Path) -> Result>> { + let mut lanes = BTreeMap::new(); + let mut unnamed = 0; + let entries = std::fs::read_dir(queue_dir) + .with_context(|| format!("reading the queue results in {}", queue_dir.display()))?; + for entry in entries { + let entry = entry.context("reading a queue results entry")?; + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let Some(lane) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if !is_lane_name(&lane) { + eprintln!("warning: a queue lane directory has an unexpected name; shown as unnamed"); + unnamed += 1; + lanes.insert(format!("unnamed-dir-{unnamed}"), Vec::new()); + continue; + } + let channels = CHANNELS + .into_iter() + .filter(|channel| regular_file(&entry.path().join(channel).join("results/report.json"))) + .collect(); + lanes.insert(lane, channels); + } + Ok(lanes) +} + +fn regular_file(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) +} + +/// What one lane contributes to the report. +#[derive(Debug, PartialEq, Eq)] +struct LanePlan { + lane: String, + blocking: bool, + phase: Option, + exit: Option, + /// Channels whose reports are copied. + reported: Vec<&'static str>, + /// Channels (or, for `None`, the lane as a whole) an empty report stands in for. + placeholders: Vec>, +} + +#[derive(Debug, PartialEq, Eq)] +struct Plan { + commit: Option, + lanes: Vec, +} + +/// Decide, lane by lane, what goes into the report. `have_matrix` says whether the +/// lane matrix could be read at all: without it nothing is known of a lane's +/// state beyond its files. +fn plan(log: &QueueLog, found: BTreeMap>, have_matrix: bool) -> Plan { + let mut names: Vec<&String> = found + .keys() + .chain(log.lanes.iter().map(|r| &r.lane)) + .collect(); + names.sort(); + names.dedup(); + let mut lanes = Vec::new(); + for name in names { + let row = log.lanes.iter().find(|r| &r.lane == name); + let reported = found.get(name).cloned().unwrap_or_default(); + let clean = row.is_some_and(MatrixRow::is_clean); + let mut placeholders = Vec::new(); + if reported.is_empty() { + // Nothing came back from this lane, whatever it was told to run. + placeholders.push(None); + } else if have_matrix && !clean { + // It did not finish cleanly, and a channel it should have run left + // no report. A lane that finished cleanly may have been told to run + // one channel only. + placeholders.extend( + CHANNELS + .into_iter() + .filter(|channel| !reported.contains(channel)) + .map(Some), + ); + } + lanes.push(LanePlan { + lane: name.clone(), + blocking: row.is_none_or(|r| r.blocking), + phase: row.and_then(|r| r.phase.clone()), + exit: row.and_then(|r| r.exit.clone()), + reported, + placeholders, + }); + } + Plan { + commit: log.commit.clone(), + lanes, + } +} + +/// Write each lane's report directories under `artifacts_dir`. +fn make_dir(dir: &Path) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display())) +} + +fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result<()> { + for lane in &plan.lanes { + for channel in &lane.reported { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + Some(channel), + )); + make_dir(&dest)?; + let src = queue_dir.join(&lane.lane).join(channel).join("results"); + for file in REPORT_FILES { + let from = src.join(file); + if !regular_file(&from) { + continue; + } + if file == "platform.json" { + // A manifest that does not parse is left out: the report then + // names the lane as having no usable manifest. + let text = std::fs::read_to_string(&from) + .with_context(|| format!("reading {}", from.display()))?; + if let Some(text) = with_lane_slug(&text, &lane.lane, channel) { + std::fs::write(dest.join(file), text) + .with_context(|| format!("writing {}", dest.join(file).display()))?; + } + } else { + std::fs::copy(&from, dest.join(file)) + .with_context(|| format!("copying {}", from.display()))?; + } + } + } + for channel in &lane.placeholders { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + *channel, + )); + make_dir(&dest)?; + // No scenarios at all: the report shows the lane as EMPTY. + std::fs::write(dest.join("report.json"), "[]\n") + .with_context(|| format!("writing {}", dest.join("report.json").display()))?; + } + } + Ok(()) +} + +/// `platform.json` with its `platform_slug` (and the copy under `capability`) +/// set to the lane's name and channel, or `None` when it is not a JSON object. +fn with_lane_slug(text: &str, lane: &str, channel: &str) -> Option { + let mut manifest: serde_json::Value = serde_json::from_str(text).ok()?; + let slug = serde_json::Value::String(format!("queue-{lane}-{channel}")); + let object = manifest.as_object_mut()?; + object.insert("platform_slug".to_owned(), slug.clone()); + if let Some(capability) = object + .get_mut("capability") + .and_then(serde_json::Value::as_object_mut) + { + capability.insert("platform_slug".to_owned(), slug); + } + serde_json::to_string_pretty(&manifest).ok() +} + +/// Text that fits in a table cell, from a reason this repository wrote itself. +fn one_line(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + ' ' + } + }) + .filter(|c| *c != '|' && *c != '`') + .take(200) + .collect::() + .trim() + .to_owned() +} + +const SECTION_HEADING: &str = "## Queue lanes\n\n"; +/// How the "nothing imported" note starts; `release-readiness` looks for it in +/// the matrix so the verdict page can say the queue lanes are missing too. +pub(crate) const ABSENT_LEAD: &str = "_No queue lanes are in this report: "; +pub(crate) const ABSENT_TAIL: &str = ". The GPUs the shared queue tests are not shown tonight._"; + +/// The section for a nightly with no queue results to import. +fn render_absent(reason: &str) -> String { + format!( + "{SECTION_HEADING}{ABSENT_LEAD}{}{ABSENT_TAIL}\n\n", + one_line(reason) + ) +} + +fn is_run_url(url: &str) -> bool { + static RUN_URL: LazyLock = LazyLock::new(|| { + Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") + .expect("static pattern") + }); + RUN_URL.is_match(url) +} + +/// The status section: which run was imported, and one row per lane. +fn render_section( + plan: &Plan, + run_url: Option<&str>, + run_started: Option<&str>, + have_matrix: bool, +) -> String { + static STARTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$").expect("static pattern") + }); + let mut out = String::from(SECTION_HEADING); + let mut source = String::from("Latest finished run of the shared GPU queue on `main`"); + if let Some(url) = run_url.filter(|u| is_run_url(u)) { + let id = url.rsplit('/').next().unwrap_or_default(); + let _ = write!(source, ": [run {id}]({url})"); + } + if let Some(started) = run_started.filter(|s| STARTED.is_match(s)) { + let _ = write!(source, ", started {started}"); + } + if let Some(commit) = &plan.commit { + let _ = write!(source, ", tested commit `{}`", &commit[..12]); + } + let _ = writeln!(out, "{source}.\n"); + if !have_matrix { + out.push_str( + "_The run's lane matrix could not be read, so every lane is shown as blocking._\n\n", + ); + } + out.push_str("| Lane | GPU | Gate | Phase | Exit | Channels reported |\n"); + out.push_str("|---|---|---|---|---|---|\n"); + for lane in &plan.lanes { + let channels = if lane.reported.is_empty() { + "none".to_owned() + } else { + lane.reported.join(", ") + }; + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {channels} |", + lane.lane, + e2e_report::queue_lane_gpu(&lane.lane), + if lane.blocking { + "blocking" + } else { + "advisory" + }, + lane.phase.as_deref().unwrap_or("unknown"), + lane.exit.as_deref().unwrap_or("unknown"), + ); + } + let _ = write!( + out, + "\nThe queue's own gate decides which lanes count: a lane it marks advisory is labelled \ + `{}` below and is left out of the release-readiness verdict, and a blocking lane that \ + reported no result is shown as `EMPTY` and counts as a failure there.\n\n", + e2e_report::ADVISORY_QUEUE_MARKER + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The lane matrix of the 2026-11-13 scheduled run, as its public job log + /// printed it. + const LOG: &str = "\u{feff}2026-11-13T02:22:14.0268419Z started gh-43351651832-1; waiting for it to finish +2026-11-13T03:45:26.1317642Z gh-43351651832-1 still running (10 min, 0 of 7 lane(s) finished) +2026-11-13T03:45:26.1323838Z request gh-43351651832-1, finished 2026-11-13T03:45:26Z +2026-11-13T03:45:26.1361153Z | image: \"rocm/dev-ubuntu-24.04:7.2.4\" +2026-11-13T03:45:26.1361751Z | blocking: \"mi300x,r9700s\" +2026-11-13T03:45:26.1362299Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351 +2026-11-13T03:45:26.1362503Z | lanes: mi300x mi325x mi350p mi350x mi355x r9700s w7900-dual +2026-11-13T03:45:26.1362739Z | +2026-11-13T03:45:26.1362910Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE +2026-11-13T03:45:26.1363201Z | mi300x rocm-cli-gh-43351651832-1-mi300x Failed 1 Exported 0m block +2026-11-13T03:45:26.1363511Z | mi325x rocm-cli-gh-43351651832-1-mi325x Failed 1 Exported 0m advisory +2026-11-13T03:45:26.1364685Z | r9700s rocm-cli-gh-43351651832-1-r9700s Failed - - 0m block +2026-11-13T03:45:26.1364991Z | w7900-dual rocm-cli-gh-43351651832-1-w7900-dual Done 0 Exported 0m advisory +2026-11-13T03:45:26.1365442Z | +2026-11-13T03:45:26.1365634Z | SUMMARY: 7 lane(s), 1 passed, 2 failed, 0 incomplete, 0 missing, 4 advisory +2026-11-13T03:45:26.1365895Z | lane mi300x: 7 file(s) collected +2026-11-13T03:45:26.1365895Z | mi999x not a row after the matrix Failed 1 Exported 0m block +"; + + fn row(lane: &str, phase: &str, exit: &str, blocking: bool) -> MatrixRow { + MatrixRow { + lane: lane.to_owned(), + phase: Some(phase.to_owned()), + exit: Some(exit.to_owned()), + blocking, + } + } + + #[test] + fn the_matrix_and_the_tested_commit_are_read_from_the_job_log() { + let log = parse_log(Some(LOG)); + assert_eq!(log.branch.as_deref(), Some("main")); + assert_eq!( + log.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + assert_eq!( + log.lanes, + vec![ + row("mi300x", "Failed", "1", true), + row("mi325x", "Failed", "1", false), + row("r9700s", "Failed", "-", true), + row("w7900-dual", "Done", "0", false), + ], + "a line after the matrix is not a lane" + ); + } + + #[test] + fn a_log_that_is_not_the_report_yields_nothing_to_publish() { + assert_eq!(parse_log(None), QueueLog::default()); + assert_eq!( + parse_log(Some( + "ERR_ACCESS_DENIED\nLANE GATE\nmi300x x\n" + )), + QueueLog::default() + ); + // A row of an unexpected shape keeps its lane and its gate, and nothing else. + let odd = "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE GATE\n\ + 2026-11-13T03:45:26Z | mi300x some workload Pending now advisory\n"; + assert_eq!( + parse_log(Some(odd)).lanes, + vec![MatrixRow { + lane: "mi300x".to_owned(), + phase: None, + exit: None, + blocking: false, + }] + ); + // Text that is not a lane name is never kept, whatever the row says; the + // lane is counted, as blocking. + let hostile = "2026-11-13T03:45:26Z | LANE GATE\n\ + 2026-11-13T03:45:26Z | ../../etc/passwd advisory\n\ + 2026-11-13T03:45:26Z | Uppercase block\n"; + let lanes = parse_log(Some(hostile)).lanes; + assert_eq!( + lanes.iter().map(|l| l.lane.as_str()).collect::>(), + ["unnamed-row-1", "unnamed-row-2"] + ); + assert!(lanes.iter().all(|l| l.blocking)); + } + + fn found(entries: &[(&str, &[&'static str])]) -> BTreeMap> { + entries + .iter() + .map(|(lane, channels)| ((*lane).to_owned(), channels.to_vec())) + .collect() + } + + #[test] + fn a_blocking_lane_with_no_result_gets_an_empty_report_and_an_advisory_one_stays_advisory() { + let log = parse_log(Some(LOG)); + let plan = plan( + &log, + found(&[ + ("mi300x", &["release"]), + ("mi325x", &["release", "nightly"]), + ("w7900-dual", &["release", "nightly"]), + ]), + true, + ); + let by_lane = |name: &str| { + plan.lanes + .iter() + .find(|l| l.lane == name) + .expect("lane planned") + }; + // Failed, and the nightly channel left nothing: one stands in for it. + assert_eq!(by_lane("mi300x").placeholders, vec![Some("nightly")]); + assert!(by_lane("mi300x").blocking); + // Nothing at all came back from r9700s, which the queue never listed a file for. + assert_eq!(by_lane("r9700s").placeholders, vec![None]); + assert!(by_lane("r9700s").reported.is_empty()); + // Advisory stays advisory, and a lane with every channel needs no stand-in. + assert!(!by_lane("mi325x").blocking); + assert!(by_lane("mi325x").placeholders.is_empty()); + assert!(by_lane("w7900-dual").placeholders.is_empty()); + assert_eq!( + plan.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + } + + #[test] + fn a_lane_that_finished_cleanly_may_have_run_one_channel_only() { + let log = QueueLog { + branch: Some("main".to_owned()), + commit: None, + lanes: vec![row("mi300x", "Done", "0", true)], + }; + let plan = plan(&log, found(&[("mi300x", &["release"])]), true); + assert!(plan.lanes[0].placeholders.is_empty()); + } + + #[test] + fn without_the_matrix_every_lane_is_blocking_and_only_a_lane_with_no_files_is_missing() { + let plan = plan( + &QueueLog::default(), + found(&[("mi325x", &["release"]), ("r9700s", &[])]), + false, + ); + assert!(plan.lanes.iter().all(|l| l.blocking)); + assert!(plan.lanes[0].placeholders.is_empty()); + assert_eq!(plan.lanes[1].placeholders, vec![None]); + } + + #[test] + fn the_lane_and_channel_replace_the_slug_that_two_lanes_share() { + let manifest = r#"{"platform_slug":"mi300x","capability":{"platform_slug":"mi300x", + "gfx_target":"gfx942"},"versions":{"channel":"release"},"expectations":[]}"#; + let rewritten = with_lane_slug(manifest, "mi325x", "nightly").expect("an object"); + let value: serde_json::Value = serde_json::from_str(&rewritten).expect("json"); + assert_eq!(value["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["gfx_target"], "gfx942"); + assert_eq!(value["versions"]["channel"], "release"); + assert_eq!(with_lane_slug("[1]", "mi325x", "release"), None); + assert_eq!(with_lane_slug("not json", "mi325x", "release"), None); + } + + fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, text).expect("write"); + } + + /// A queue artifact as the run above published it, cut down to two lanes. + fn queue_artifact(dir: &Path) { + for (lane, slug, channels) in [ + ("mi300x", "mi300x", &["release"][..]), + ("mi325x", "mi300x", &["release", "nightly"][..]), + ] { + write(&dir.join(lane).join("lane.log"), "the lane's own log\n"); + for channel in channels { + let results = dir.join(lane).join(channel).join("results"); + write(&results.join("report.json"), "[]\n"); + write(&results.join("commands.jsonl"), ""); + write(&results.join("junit.xml"), ""); + write(&results.join("harness-diagnostics.log"), "diagnostics"); + write( + &results.join("platform.json"), + &format!( + r#"{{"platform_slug":"{slug}","versions":{{"channel":"{channel}"}},"expectations":[]}}"# + ), + ); + } + } + write(&dir.join("r9700s").join("lane.log"), "no files came back\n"); + } + + #[test] + fn the_import_writes_only_what_the_report_reads_under_names_that_say_lane_and_gate() { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let log = queue.path().join("job.log"); + // The log sits beside the lanes here; a stray file is no lane. + write(&log, LOG); + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("consolidated/queue-lanes.md"); + let args = Args { + queue_dir: Some(queue.path().to_owned()), + log: Some(log), + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/43351651832".to_owned()), + run_started: Some("2026-11-13T02:21:58Z".to_owned()), + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }; + run(&args).expect("import"); + + let mut dirs: Vec = std::fs::read_dir(out.path().join("e2e-artifacts")) + .expect("artifacts") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + dirs.sort(); + assert_eq!( + dirs, + [ + "e2e-queue-mi300x-nightly-report", + "e2e-queue-mi300x-release-report", + "e2e-queue-mi325x-advisory-nightly-report", + "e2e-queue-mi325x-advisory-release-report", + "e2e-queue-r9700s-report", + // Listed in the log, with no files at all: an empty stand-in. + "e2e-queue-w7900-dual-advisory-report", + ] + ); + let copied = out + .path() + .join("e2e-artifacts/e2e-queue-mi325x-advisory-release-report"); + let mut files: Vec = std::fs::read_dir(&copied) + .expect("lane dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + files.sort(); + assert_eq!(files, ["commands.jsonl", "platform.json", "report.json"]); + let manifest = std::fs::read_to_string(copied.join("platform.json")).expect("manifest"); + assert!(manifest.contains("queue-mi325x-release"), "{manifest}"); + assert_eq!( + std::fs::read_to_string( + out.path() + .join("e2e-artifacts/e2e-queue-r9700s-report/report.json") + ) + .expect("placeholder"), + "[]\n" + ); + + let section = std::fs::read_to_string(section).expect("section"); + assert!(section.starts_with("## Queue lanes\n"), "{section}"); + assert!( + section.contains( + "[run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832), \ + started 2026-11-13T02:21:58Z, tested commit `691834a04540`" + ), + "{section}" + ); + assert!( + section.contains("| mi325x | MI325X | advisory | Failed | 1 | release, nightly |"), + "{section}" + ); + assert!( + section.contains("| r9700s | R9700 | blocking | Failed | - | none |"), + "{section}" + ); + // Nothing of the lane's own log or the workload names reaches the page. + assert!(!section.contains("rocm-cli-gh-"), "{section}"); + assert!(!section.contains("lane's own log"), "{section}"); + } + + #[test] + fn an_absent_queue_run_is_a_section_that_says_so_and_nothing_else() { + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: None, + log: None, + run_url: None, + run_started: None, + absent: Some("no queue run finished on main in the last 24 hours | `x`\n".to_owned()), + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("an absent run is not an error"); + assert!(!out.path().join("e2e-artifacts").exists()); + let text = std::fs::read_to_string(section).expect("section"); + assert_eq!( + text, + "## Queue lanes\n\n_No queue lanes are in this report: no queue run finished on \ + main in the last 24 hours x. The GPUs the shared queue tests are not shown \ + tonight._\n\n" + ); + } + + fn import_with_log(log_text: Option<&str>) -> (tempfile::TempDir, String) { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let out = tempfile::tempdir().expect("tempdir"); + let log = log_text.map(|text| { + let path = out.path().join("job.log"); + write(&path, text); + path + }); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: Some(queue.path().to_owned()), + log, + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/1".to_owned()), + run_started: None, + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("a log that cannot be used is not an error"); + let text = std::fs::read_to_string(section).expect("section"); + (out, text) + } + + #[test] + fn a_run_that_tested_another_branch_or_whose_log_is_unreadable_is_not_imported() { + // A run started from main can be asked to test any branch of the repository. + let other = LOG.replace("rocm-cli main at", "rocm-cli some-branch at"); + for log in [ + Some(other.as_str()), + // What the egress proxy answers when it refuses the request. + Some("ERR_ACCESS_DENIED\n"), + None, + ] { + let (out, text) = import_with_log(log); + assert!( + text.contains("did not test main, or its job log could not be read"), + "{text}" + ); + assert!(!text.contains("| Lane |"), "{text}"); + assert!(!out.path().join("e2e-artifacts").exists()); + } + } + + #[test] + fn a_log_without_a_matrix_leaves_every_lane_blocking_and_says_so() { + let (out, text) = import_with_log(Some( + "2026-11-13T03:45:26Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351\n", + )); + assert!(text.contains("every lane is shown as blocking"), "{text}"); + assert!(!text.contains("advisory |"), "{text}"); + // The lane with no files is still a missing result; the others are not + // second-guessed about a channel they never ran. + let arts = out.path().join("e2e-artifacts"); + assert!(arts.join("e2e-queue-r9700s-report").is_dir()); + assert!(!arts.join("e2e-queue-mi300x-nightly-report").exists()); + } + + #[test] + fn a_lane_that_cannot_be_named_still_shows_as_a_blocking_lane_with_no_result() { + for name in [ + "Mi300X", + "mi300x_2", + "x-advisory", + "x-release", + "x-nightly", + "-x", + ] { + assert!(!is_lane_name(name), "{name}"); + } + let log = parse_log(Some( + "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE\n\ + 2026-11-13T03:45:26Z | x-advisory w Done 0 Exported 0m advisory\n", + )); + assert_eq!( + log.lanes, + vec![MatrixRow { + lane: "unnamed-row-1".to_owned(), + phase: None, + exit: None, + blocking: true, + }] + ); + // The same for a directory of the artifact. + let queue = tempfile::tempdir().expect("tempdir"); + write(&queue.path().join("mi300x_2/lane.log"), "log\n"); + write( + &queue.path().join("mi300x/release/results/report.json"), + "[]", + ); + let found = scan(queue.path()).expect("scan"); + assert_eq!(found.get("unnamed-dir-1"), Some(&Vec::new())); + assert_eq!(found.get("mi300x"), Some(&vec!["release"])); + let plan = plan(&QueueLog::default(), found, false); + let unnamed = plan + .lanes + .iter() + .find(|l| l.lane == "unnamed-dir-1") + .expect("lane"); + assert!(unnamed.blocking); + assert_eq!(unnamed.placeholders, vec![None]); + } + + #[test] + fn a_run_url_that_is_not_a_run_is_not_linked() { + let plan = Plan { + commit: None, + lanes: Vec::new(), + }; + let text = render_section(&plan, Some("https://example.com/x)[y]("), None, true); + assert!(!text.contains("example.com"), "{text}"); + } +} diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 63fa727cc5..73b0001c81 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -202,6 +202,7 @@ mod tests { const CANONICAL_REPORT_ARTIFACTS: &[&str] = &[ "e2e-report", "e2e-gpu-report", + "e2e-gpu-selfhosted-report", "e2e-gpu-n-1-report", "e2e-gpu-n-2-report", "e2e-gpu-rad3-report", @@ -299,7 +300,10 @@ mod tests { // its own breadth-over-versions tradeoff -- so strip the `-n-1`/ // `-n-2` legs before comparing; the `current` leg's name is // unaffected by the matrix and must still match nightly's. - for suffix in ["-n-1-report", "-n-2-report"] { + // `-selfhosted` is the same MI300X silicon on a self-hosted runner, + // beside the queue's `e2e-gpu-report` on a pull request; nightly + // runs the one MI300X lane under that base name. + for suffix in ["-n-1-report", "-n-2-report", "-selfhosted-report"] { for name in &mut names { if let Some(rest) = name.strip_suffix(suffix) { *name = format!("{rest}-report"); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e3db2768f6..0fda6b30ec 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,8 @@ mod dash_text_edit_contract; mod demos; mod e2e; mod e2e_prewarm; +mod e2e_prewarm_vision; +mod e2e_queue_import; mod e2e_report; mod engine_stdio_client_contract; mod env_mutation_contract; @@ -331,6 +333,12 @@ pub(crate) enum Command { #[arg(long)] html_out: PathBuf, }, + /// Bring a finished run of the shared GPU queue into the nightly's E2E + /// report: write each lane's reports where `e2e-report` reads them, named + /// for the lane and for whether the queue blocks on it, and write a status + /// section for the top of the support matrix. With `--absent`, import + /// nothing and say why. See docs/nightly-on-the-queue.md. + E2eQueueImport(e2e_queue_import::Args), /// Count received Doctor reports from an export of the report mailbox. /// /// Reads `.eml` files (one message each) and raw `.json` report bodies, @@ -497,6 +505,7 @@ fn run() -> Result<()> { artifacts_dir, html_out, } => e2e_report::run(&artifacts_dir, &html_out)?, + Command::E2eQueueImport(args) => e2e_queue_import::run(&args)?, Command::ReportRollup(args) => report_rollup::run(&args)?, Command::ReleaseReadiness { since_tag, diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index c3060f5574..1c85f2e26b 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -72,6 +72,11 @@ pub struct Facts { /// Whether the nightly or Windows build gate was skipped (a legitimate /// no-op-night outcome, not a failure) rather than actually run. pub build_skipped: bool, + /// Why the shared queue's GPU lanes are not in tonight's E2E report, when + /// the matrix says so. The verdict does not count that absence as a + /// failure, so the table has to say it: the E2E row covers only the lanes + /// that are in the report. + pub queue_absent: Option, pub commits: CommitBuckets, pub known_issues: Vec, } @@ -93,7 +98,7 @@ pub fn run( .with_context(|| format!("reading e2e report {}", e2e_report.display()))?; let e2e_fail_total = total_e2e_failures(&e2e_text); - let facts = assemble_facts( + let mut facts = assemble_facts( since_tag, commits, known_issues, @@ -102,6 +107,7 @@ pub fn run( windows_result, e2e_report_job_result, ); + facts.note_queue_absent(queue_absent_reason(&e2e_text)); // Trailing newline so the file ends like a text file. write_parented(out, &format!("{}\n", serde_json::to_string_pretty(&facts)?))?; @@ -164,11 +170,24 @@ fn assemble_facts( failing_gates, e2e_fail_total, build_skipped: nightly_result == "skipped" || windows_result == "skipped", + queue_absent: None, commits, known_issues, } } +impl Facts { + /// Record that the queue's lanes are not in the report. A queue that never + /// ran is not a pass, so this is a failing gate: the verdict is NO-GO. + fn note_queue_absent(&mut self, reason: Option) { + let Some(reason) = reason else { return }; + self.failing_gates + .push(format!("queue lanes absent: {reason}")); + self.verdict = Verdict::NoGo; + self.queue_absent = Some(reason); + } +} + fn write_parented(path: &Path, contents: &str) -> Result<()> { if let Some(parent) = path.parent() && !parent.as_os_str().is_empty() @@ -393,19 +412,34 @@ fn excerpt(text: &str, max_chars: usize) -> String { } } +/// The reason the matrix gives for having no queue lanes, if it says so +/// (`e2e_queue_import::render_absent`). +fn queue_absent_reason(markdown: &str) -> Option { + use crate::e2e_queue_import::{ABSENT_LEAD, ABSENT_TAIL}; + markdown.lines().find_map(|line| { + let rest = line.strip_prefix(ABSENT_LEAD)?; + Some(rest.strip_suffix(ABSENT_TAIL).unwrap_or(rest).to_string()) + }) +} + /// Sum of the `Fail` column across the consolidated E2E markdown's `**Total**` /// row (`crates/e2e-report`'s `consolidated_summary_markdown` format: `| /// Platform | OS | Total | Pass | Fail | Skip | Xfail | Status |`), plus one /// for every per-platform row that reports a non-`PASS` `Status` with a `Fail` /// cell of `0`. /// -/// The second part exists because an `EMPTY` platform (no `report.json`, or -/// one that's empty/unparseable) contributes `0` to every numeric column — -/// including `Fail` — so the Total row's own Fail cell has no record that the -/// platform was ever there at all. A genuine `FAIL` status always has a -/// nonzero own `Fail` cell (already folded into the Total row), so this only -/// adds for platforms the Total row would otherwise read as having zero -/// failures. +/// The second part exists because two kinds of failed row contribute `0` to +/// the `Fail` column, so the Total row's own Fail cell has no record of them: +/// an `EMPTY` platform (no `report.json`, or one that's empty/unparseable), +/// and a `FAIL` platform whose only problem is the harness's post-run lane +/// check (`lane_failure` in its `platform.json`, e.g. "0 scenarios executed"), +/// which fails no scenario cell. A `FAIL` row with a nonzero own `Fail` cell +/// is already folded into the Total row and is not counted again. +/// +/// A row for a shared-queue lane the queue does not block on (its platform +/// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the +/// queue lets that lane fail, so the verdict does too. Its `Fail` cell is +/// taken back out of the Total row, which sums every row. /// /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail @@ -431,24 +465,31 @@ fn total_e2e_failures(markdown: &str) -> Option { .nth(5) .and_then(|cell| cell.trim().parse::().ok())?; - let silently_empty_platforms = lines[header_idx + 1..total_idx] - .iter() - .filter(|l| { - let t = l.trim_start(); - t.starts_with('|') && !t.starts_with("|---") - }) - .filter(|l| { - let cells: Vec<&str> = l.split('|').collect(); - let status = cells.get(8).map(|c| c.trim()); - let fail = cells - .get(5) - .and_then(|c| c.trim().parse::().ok()) - .unwrap_or(0); - !matches!(status, Some("PASS") | None) && fail == 0 - }) - .count() as u64; + let mut advisory_fails = 0; + let mut silently_empty_platforms = 0; + for line in lines[header_idx + 1..total_idx].iter().filter(|l| { + let t = l.trim_start(); + t.starts_with('|') && !t.starts_with("|---") + }) { + let cells: Vec<&str> = line.split('|').collect(); + let fail = cells + .get(5) + .and_then(|c| c.trim().parse::().ok()) + .unwrap_or(0); + if cells + .get(1) + .is_some_and(|c| c.contains(e2e_report::ADVISORY_QUEUE_MARKER)) + { + advisory_fails += fail; + continue; + } + let status = cells.get(8).map(|c| c.trim()); + if !matches!(status, Some("PASS") | None) && fail == 0 { + silently_empty_platforms += 1; + } + } - Some(total_row_fails + silently_empty_platforms) + Some(total_row_fails.saturating_sub(advisory_fails) + silently_empty_platforms) } /// Neutralize markdown syntax in untrusted text (GitHub issue titles) before @@ -485,10 +526,13 @@ pub(crate) fn render_table(facts: &Facts) -> String { // the header below so `publish-release-readiness-wiki`'s guard (which // greps for it) only trips on a truly empty/degenerate table. let e2e_failing = facts.failing_gates.iter().any(|g| g.starts_with("E2E")); - let e2e_evidence = match facts.e2e_fail_total { + let mut e2e_evidence = match facts.e2e_fail_total { Some(n) => format!("{n} failure(s) across all platforms"), None => "evidence unreadable".to_string(), }; + if facts.queue_absent.is_some() { + e2e_evidence.push_str(" in the report (queue lanes missing, see below)"); + } out.push_str("| Area | Evidence | Status |\n|---|---|:--:|\n"); let _ = writeln!( @@ -509,6 +553,16 @@ pub(crate) fn render_table(facts: &Facts) -> String { ); out.push('\n'); + if let Some(reason) = &facts.queue_absent { + let _ = writeln!( + out, + "⚠️ **The shared queue's GPU lanes are not in tonight's report** ({}). The E2E row \ + above does not cover R9700, MI350P, W7900 or the queue's blocking lanes, and a queue \ + that did not run is not a pass: the verdict is NO-GO.\n", + escape_markdown(reason) + ); + } + // The status table above only has room for a glyph per row, which is // ambiguous on its own (e.g. a cancelled E2E lane renders as // "0 failure(s)" with a red ❌, which looks contradictory). Spell out @@ -741,6 +795,96 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + /// A lane the harness failed after its run with no failed scenario cell + /// (every scenario passed or was skipped) reads `FAIL` with a `Fail` cell + /// of 0; the Total row cannot show it, so the row itself must count. One + /// that also has a failed cell is already in the Total row and is counted + /// once, not twice. + #[test] + fn total_e2e_failures_counts_a_lane_check_failure_with_no_failed_cells() { + // Real renderer output over a platform.json that records a lane + // failure, with the one scenario passing or failing. + let render = |step_status: &str| { + let dir = tempfile::tempdir().expect("tempdir"); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s0","tags":[{{"name":"id:examine-a"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{step_status}"}}}}]}}]}}]"# + ), + ) + .expect("write report"); + std::fs::write( + dir.path().join("platform.json"), + r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#, + ) + .expect("write platform.json"); + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&[( + "e2e-gpu-report".to_string(), + report, + )])) + }; + + let md = render("passed"); + assert!( + md.contains("| 1 | 1 | 0 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 1 | 0 |"), + "expected a FAIL row with a Fail cell of 0:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + + let md = render("failed"); + assert!( + md.contains("| 1 | 0 | 1 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 0 | 1 |"), + "expected a FAIL row with a Fail cell of 1:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + } + + #[test] + fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { + // Real producer output, so the marker the parser looks for is the one the + // report writes: a failing advisory lane and an empty advisory lane add + // nothing, while the same lanes marked blocking add one failure and one + // silent failure. + fn report(status: &str) -> String { + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s","tags":[],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{status}"}}}}]}}]}}]"# + ) + } + let render = |advisory: bool| { + let dir = tempfile::tempdir().expect("tempdir"); + let write = |name: &str, text: &str| { + let path = dir.path().join(name); + std::fs::create_dir_all(&path).expect("mkdir"); + std::fs::write(path.join("report.json"), text).expect("write report"); + (name.to_owned(), path.join("report.json")) + }; + let failing = e2e_report::queue_artifact_name("mi325x", advisory, Some("release")); + let empty = e2e_report::queue_artifact_name("r9700s", advisory, None); + let inputs = vec![ + write("e2e-gpu-report", &report("passed")), + write(&failing, &report("failed")), + write(&empty, "[]"), + ]; + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&inputs)) + }; + let advisory = render(true); + assert!( + advisory.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{advisory}" + ); + assert_eq!(total_e2e_failures(&advisory), Some(0), "{advisory}"); + let blocking = render(false); + assert!( + !blocking.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{blocking}" + ); + assert_eq!(total_e2e_failures(&blocking), Some(2), "{blocking}"); + } + #[test] fn total_e2e_failures_ignores_the_expectation_grid_and_command_coverage_tables() { // Regression: the scan used to run over every `|`-prefixed line in the @@ -797,6 +941,49 @@ mod tests { ); } + #[test] + fn the_table_says_when_the_queue_lanes_are_missing() { + let md = format!( + "## Queue lanes\n\n{}{}{}\n\n", + crate::e2e_queue_import::ABSENT_LEAD, + "no queue run on main finished in the last 24 hours", + crate::e2e_queue_import::ABSENT_TAIL + ); + let reason = queue_absent_reason(&md).expect("the note is recognised"); + assert_eq!(reason, "no queue run on main finished in the last 24 hours"); + assert_eq!(queue_absent_reason("| Total | 0 |\n"), None); + let mut facts = assemble_facts( + "v1.0.0", + CommitBuckets::default(), + vec![], + Some(0), + "success", + "success", + "success", + ); + assert_eq!(facts.verdict, Verdict::Go); + facts.note_queue_absent(Some(reason.clone())); + let table = render_table(&facts); + assert!(table.contains("queue lanes missing"), "{table}"); + assert!(table.contains("not in tonight's report"), "{table}"); + // A queue that never ran is not a pass. + assert_eq!(facts.verdict, Verdict::NoGo); + assert_eq!( + facts.failing_gates, + [format!("queue lanes absent: {reason}")] + ); + assert!(table.contains("NO-GO"), "{table}"); + assert!( + table.contains("queue lanes absent: no queue run"), + "{table}" + ); + // Nothing absent, nothing changed. + facts.verdict = Verdict::Go; + facts.failing_gates.clear(); + facts.note_queue_absent(None); + assert_eq!(facts.verdict, Verdict::Go); + } + #[test] fn assemble_facts_is_go_when_everything_succeeds() { let facts = assemble_facts( diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 752cf97a76..662382c731 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -9,8 +9,9 @@ //! This closes the gap where CI verifies release artifacts against the public key //! configured in its environment (see `scripts/release_readiness.py`) while //! installers and the binary trust a *separately embedded* copy. If those diverge, -//! CI can bless a release that installers then reject once default-on verification -//! lands. This check enforces a single source of truth: the committed +//! CI can bless a release that installers then reject, since installers verify +//! every install against the pinned release key unless the user supplies +//! another. This check enforces a single source of truth: the committed //! `docs/keys/*.pem` files. //! //! Design notes: diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b0..120d60bc31 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1718,6 +1718,594 @@ jobs: } } + /// Self-hosted E2E lanes that declare no usable GPU, each with the reason. + /// Every other self-hosted lane must declare one. + const NO_GPU_LANES: [(&str, &str); 2] = [ + ( + "e2e-wsl", + "the WSL distro is deliberately left without ROCm's WSL driver bridge, \ + which `rocm install driver` installs under test", + ), + ( + "e2e-wsl-nightly", + "the nightly twin of e2e-wsl, set up the same way", + ), + ]; + + /// Whether a step's `run:` body invokes the E2E suite itself + /// (`cargo xtask e2e`, not `e2e-prewarm` or `e2e-report`). + fn runs_the_e2e_suite(step: &str) -> bool { + run_block(step).is_some_and(|body| { + body.iter().any(|line| { + line.match_indices("cargo xtask e2e").any(|(at, m)| { + line[at + m.len()..] + .chars() + .next() + .is_none_or(char::is_whitespace) + }) + }) + }) + } + + /// Every self-hosted E2E lane declares whether it has a GPU, so the harness + /// can fail a GPU lane whose probe found none instead of passing it with + /// every GPU scenario (`@requires-gpu` or `@requires-multi-gpu`) skipped. + /// + /// Derived from the workflows like the GPU-preflight check above, so a new + /// lane is held to it the day it lands. The declaration is read from the + /// job-level `env:` mapping: a step-level one would leave the other steps, + /// and a reader of the job header, without it, and one on the step that + /// runs the suite would override it there. A lane that runs the suite + /// inside WSL must name the variable in every `WSLENV` it sets, and in one + /// on the step that runs the suite, since `wsl.exe` passes nothing else + /// into the guest. + #[test] + fn every_self_hosted_lane_declares_whether_it_has_a_gpu() { + let name = e2e_report::EXPECT_GPU_ENV; + let mut no_gpu_seen = Vec::new(); + for (workflow, text) in self_hosted_workflows() { + for (job, _) in self_hosted_e2e_jobs(&text) { + let block = job_block(&text, &job); + let env = job_mapping(block, "env"); + let exempt = NO_GPU_LANES.iter().any(|(lane, _)| *lane == job); + match env.get(name).map(String::as_str) { + Some("1") => assert!( + !exempt, + "{workflow} job `{job}` is listed in NO_GPU_LANES but declares \ + {name}: \"1\"; drop it from the list" + ), + Some("0") => { + assert!( + exempt, + "{workflow} job `{job}` declares {name}: \"0\" but is not in \ + NO_GPU_LANES; a self-hosted lane without a GPU needs its reason \ + recorded there" + ); + no_gpu_seen.push(job.clone()); + } + other => panic!( + "{workflow} job `{job}` runs on self-hosted hardware but its job-level \ + `env:` sets {name} to {other:?}; it must be \"1\" (a GPU lane) or \ + \"0\" (listed in NO_GPU_LANES)" + ), + } + let why = NO_GPU_LANES + .iter() + .find(|(lane, _)| *lane == job) + .map_or_else(String::new, |(_, reason)| { + format!(" (a NO_GPU_LANES lane: {reason})") + }); + let names_it = |wslenv: &str| { + wslenv + .trim_matches(|c| c == '\'' || c == '"') + .split(':') + .any(|var| var.split('/').next() == Some(name)) + }; + for wslenv in block + .lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + { + assert!( + names_it(wslenv), + "{workflow} job `{job}`{why} sets WSLENV without {name}, so the suite \ + inside WSL reads the lane as undeclared: {wslenv}" + ); + } + // The step that runs the suite is where the declaration has + // to arrive: a step-level override there would replace the + // job's value, and a WSL step that names it in no WSLENV of + // its own leaves the guest without it, whatever other steps do. + let steps = job_steps(block) + .unwrap_or_else(|| panic!("{workflow} job `{job}` has no `steps:` list")); + let harness: Vec<&(String, String)> = steps + .iter() + .filter(|(_, step)| runs_the_e2e_suite(step)) + .collect(); + assert!( + !harness.is_empty(), + "{workflow} job `{job}` has no step that runs `cargo xtask e2e`" + ); + for (step_name, step) in harness { + assert!( + !step_env(step).contains_key(name), + "{workflow} job `{job}`{why}: step `{step_name}` sets {name} itself, \ + overriding the job-level declaration" + ); + if step.contains("Invoke-WslBash.ps1") { + assert!( + step.lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + .any(names_it), + "{workflow} job `{job}`{why}: step `{step_name}` runs the suite in \ + WSL but sets no WSLENV naming {name}, so the guest reads the lane \ + as undeclared" + ); + } + } + } + } + for (lane, _) in NO_GPU_LANES { + assert!( + no_gpu_seen.iter().any(|seen| seen == lane), + "NO_GPU_LANES lists `{lane}`, which no self-hosted workflow declares \ + {name}: \"0\"; drop the stale entry" + ); + } + // The queue lanes run through this script, not a workflow job; every + // queue lane is given a GPU. + let script = std::fs::read_to_string(repo_root().join("scripts/run-e2e-lane.sh")) + .expect("reading the lane script"); + let export = format!("export {name}=\"${{{name}-1}}\""); + assert!( + script.lines().any(|line| line.trim() == export), + "scripts/run-e2e-lane.sh must declare its lanes' GPU with `{export}`" + ); + } + + #[test] + fn only_e2e_gpu_may_carry_the_stale_gate_on_its_preflight() { + let workflow = |job: &str| { + format!( + "jobs:\n {job}:\n steps:\n - name: GPU preflight (x)\n {STALE_RUN_GATE}\n run: |\n bash p.sh\n" + ) + }; + let stripped = gpu_preflight_steps(&workflow("e2e-gpu")); + assert_eq!(stripped.len(), 1); + assert!( + !stripped[0].contains(STALE_RUN_GATE), + "e2e-gpu's gate is removed so the shape checks see a bare delegation" + ); + for job in ["e2e-gpu-nightly", "e2e-wsl", "e2e-gpu-strix-ubuntu"] { + let kept = gpu_preflight_steps(&workflow(job)); + assert!( + kept[0].contains(STALE_RUN_GATE), + "`{job}`'s preflight keeps an `if:`, so the shape checks still reject it" + ); + } + } + + #[test] + fn mi300x_lane_skips_itself_when_its_run_is_stale() { + // A queued MI300X job for a superseded head or a closed PR otherwise + // costs ~45 min of runner time once it starts, and `gh run cancel` from + // outside does not always take it off the queue. The job-level `if` + // cannot see staleness (evaluated before queueing), so the first step + // records it and every later step is gated on that output; the cancel + // itself is `cancel-stale-run`'s (pinned in the test below). + let text = read_workflow("e2e-selfhosted.yml"); + let block = job_block(&text, "e2e-gpu"); + + assert_eq!( + permission_mapping(block, " permissions:"), + BTreeMap::from([ + ("contents".to_owned(), "read".to_owned()), + ("pull-requests".to_owned(), "read".to_owned()), + ]), + "e2e-gpu runs PR code on a persistent runner: it holds read scopes only, \ + and the write-scoped cancel lives in `cancel-stale-run`" + ); + assert_eq!( + nested_block(block, " outputs:") + .lines() + .skip(1) + .map(str::trim) + .filter(|l| !l.starts_with('#') && !l.is_empty()) + .collect::>(), + ["stale: ${{ steps.stale.outputs.stale }}"], + "e2e-gpu must publish exactly the guard's `stale` output for `e2e-report`" + ); + + let steps = job_steps(block).expect("e2e-gpu has steps"); + let names: Vec<&str> = steps.iter().map(|(n, _)| n.as_str()).collect(); + for required in [ + "Reclaim GPU from stray E2E processes", + "Run E2E tests on GPU hardware", + "Upload E2E report (current)", + "Upload E2E report (n-1)", + "Upload E2E report (n-2)", + ] { + assert!( + names.contains(&required), + "e2e-gpu lost its `{required}` step: {names:?}" + ); + } + let (_, guard) = &steps[0]; + assert_eq!( + step_key(guard, "id"), + "stale", + "the first e2e-gpu step is the stale-run guard" + ); + assert_eq!( + step_key(guard, "if"), + "github.event_name == 'pull_request'", + "the guard runs on pull_request only" + ); + assert_eq!( + step_env(guard), + BTreeMap::from([ + ("GH_TOKEN".to_owned(), "${{ github.token }}".to_owned()), + ("GH_REPO".to_owned(), "${{ github.repository }}".to_owned()), + ( + "PR_NUMBER".to_owned(), + "${{ github.event.pull_request.number }}".to_owned() + ), + // The event's head, not the merge ref (`github.sha`), which would + // read as stale on every run. + ( + "EVENT_HEAD_SHA".to_owned(), + "${{ github.event.pull_request.head.sha }}".to_owned() + ), + ]), + "the guard's environment is pinned exactly" + ); + + // Every later step carries exactly the gate, uploads ANDed with their + // existing condition. Exact comparison, not `contains`: `... && false` + // or `always() || ...` would still contain the gate. + for (name, step) in &steps[1..] { + let cond = step_key(step, "if"); + let ok = if name.starts_with("Upload E2E report (") { + let leg = name + .strip_prefix("Upload E2E report (") + .and_then(|n| n.strip_suffix(')')) + .unwrap(); + cond == format!( + "always() && steps.stale.outputs.stale != 'true' && matrix.sdk_version == '{leg}'" + ) + } else { + cond == STALE_RUN_GATE.strip_prefix("if: ").unwrap() + }; + assert!( + ok, + "e2e-gpu step `{name}` must carry exactly the stale gate in its `if:` \ + (found `{cond}`)" + ); + } + } + + #[test] + fn stale_runs_are_cancelled_by_a_hosted_job_that_runs_no_pr_code() { + // `actions: write` can dispatch workflows. It must never share a job + // with PR code on the persistent MI300X runner (where a write scope + // would sit next to untrusted dependency code), so the cancel + // is its own job: hosted, no checkout, same-repo PRs only. + let text = read_workflow("e2e-selfhosted.yml"); + let block = job_block(&text, "cancel-stale-run"); + assert_eq!( + permission_mapping(block, " permissions:"), + BTreeMap::from([("actions".to_owned(), "write".to_owned())]), + "the cancel job holds exactly `actions: write`" + ); + assert_eq!( + job_scalar(block, "runs-on"), + "ubuntu-24.04", + "hosted, never self-hosted" + ); + assert_eq!(job_scalar(block, "needs"), "[e2e-gpu]"); + assert_eq!(job_scalar(block, "timeout-minutes"), "5"); + let cond = normalized_whitespace( + &block + .lines() + .skip_while(|l| l.trim() != "if: >-") + .skip(1) + .take_while(|l| indent_of(l) > indent_of(" if: >-")) + .map(str::trim) + .collect::>() + .join(" "), + ); + assert_eq!( + cond, + "always() && github.event_name == 'pull_request' && \ + github.event.pull_request.head.repo.full_name == github.repository && \ + needs.e2e-gpu.outputs.stale == 'true'", + "only a same-repo pull_request whose e2e-gpu found it stale is cancelled" + ); + let steps = job_steps(block).expect("the cancel job has steps"); + assert_eq!( + steps.len(), + 1, + "one step: the cancel, no checkout or action" + ); + assert!( + !steps[0].1.contains("uses:"), + "the write-scoped job runs no third-party action" + ); + assert_eq!( + step_env(&steps[0].1), + BTreeMap::from([ + ("GH_TOKEN".to_owned(), "${{ github.token }}".to_owned()), + ("GH_REPO".to_owned(), "${{ github.repository }}".to_owned()), + ]), + "the cancel step's environment is pinned exactly" + ); + // The step is exactly one command, so nothing else (a clone, a script + // from the PR) can ride along in the job that holds `actions: write`. + let script = run_script(&steps[0].1).expect("run block"); + let script: Vec<&str> = script + .iter() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect(); + assert_eq!( + script, + [ + "gh run cancel \"$GITHUB_RUN_ID\" || echo \"::warning::could not cancel run ${GITHUB_RUN_ID}; its other lanes keep running\"" + ], + "the cancel step is one command and tolerates a failed cancel" + ); + } + + #[test] + #[cfg(unix)] + fn the_cancel_step_cancels_this_run_and_survives_a_refusal() { + let text = read_workflow("e2e-selfhosted.yml"); + let steps = job_steps(job_block(&text, "cancel-stale-run")).expect("steps"); + let step = &steps[0].1; + for (refuses, expect_warning) in [(false, false), (true, true)] { + let dir = tempfile::tempdir().expect("temp dir"); + let script = dir.path().join("cancel.sh"); + let calls = dir.path().join("calls"); + std::fs::write( + &script, + dedent(&run_block(step).expect("run block")).expect("non-empty"), + ) + .expect("write script"); + let stub = "#!/bin/sh\n\ +[ -n \"$GH_TOKEN\" ] && [ \"$GH_REPO\" = owner/repo ] || exit 1\n\ +echo \"$*\" >> \"$STUB_CALLS\"\n\ +[ -z \"$STUB_REFUSES\" ]\n"; + rocm_core::fake_executable::write_fake_executable(&dir.path().join("gh"), stub) + .expect("plant gh stub"); + let path = format!( + "{}:{}", + dir.path().display(), + std::env::var("PATH").unwrap_or_default() + ); + let mut cmd = std::process::Command::new("/bin/bash"); + cmd.arg("-e") + .arg(&script) + .env("PATH", path) + .env("GITHUB_RUN_ID", "42") + .env("STUB_CALLS", &calls); + for key in step_env(step).keys() { + cmd.env( + key, + if key == "GH_REPO" { + "owner/repo" + } else { + "token" + }, + ); + } + if refuses { + cmd.env("STUB_REFUSES", "1"); + } + let out = cmd.output().expect("running the cancel step"); + let log = String::from_utf8_lossy(&out.stdout).into_owned(); + assert!( + out.status.success(), + "a refused cancel must not fail the job: {log}" + ); + assert_eq!( + std::fs::read_to_string(&calls).unwrap_or_default().trim(), + "run cancel 42" + ); + assert_eq!( + log.contains("::warning::could not cancel run 42"), + expect_warning + ); + } + } + + /// How the stub `gh` answers the guard's PR lookup. + #[cfg(unix)] + #[derive(Clone, Copy)] + enum PrRead { + /// The PR with this state and head sha. + Pr(&'static str, &'static str), + /// `gh api` exits non-zero. + Fails, + /// `gh api` exits 0 and prints nothing. + Empty, + /// `gh api` exits 0 on a PR whose state and head are both null. + Nulls, + /// The PR has a state but a null head sha. + NoHead, + /// The PR has a null state but a head sha. + NoState, + /// There is no `gh` on the PATH at all. + NoGh, + } + + /// Run the stale-run guard's script under `bash -e` with a stub `gh`, as a + /// `run:` step does; returns (exit code, `$GITHUB_OUTPUT`, log, summary). + /// + /// The child's environment is built from the step's own `env:` keys, so a + /// key deleted from the workflow is missing here too. The stub rejects any + /// endpoint but `repos/owner/repo/pulls/7`, any call without a token, and + /// any `gh` subcommand but `api`: the guard has no business cancelling. + #[cfg(unix)] + fn run_stale_guard(read: PrRead) -> (i32, String, String, String) { + let text = read_workflow("e2e-selfhosted.yml"); + let steps = job_steps(job_block(&text, "e2e-gpu")).expect("e2e-gpu has steps"); + let guard = &steps[0].1; + let dir = tempfile::tempdir().expect("temp dir"); + let script = dir.path().join("guard.sh"); + let output = dir.path().join("output"); + let summary = dir.path().join("summary.md"); + std::fs::write( + &script, + dedent(&run_block(guard).expect("the guard has a `run: |` block")) + .expect("the guard script is not empty"), + ) + .expect("write script"); + std::fs::write(&output, "").expect("write output"); + std::fs::write(&summary, "").expect("write summary"); + // The stub answers `gh api ... --jq EXPR` by running the real `jq` over a + // canned PR object, so a wrong field in the guard's query is caught too. + let stub = "#!/bin/sh\n\ +[ \"$1\" = api ] || { echo \"unexpected gh $*\" >&2; exit 99; }\n\ +[ -n \"$GH_TOKEN\" ] || exit 1\n\ +[ \"$2\" = repos/owner/repo/pulls/7 ] || exit 1\n\ +case \"$STUB_MODE\" in\n\ + fails) exit 1 ;;\n\ + empty) exit 0 ;;\n\ +esac\n\ +while [ $# -gt 0 ]; do [ \"$1\" = --jq ] && q=$2; shift; done\n\ +case \"$STUB_MODE\" in\n\ + nulls) pr='{\"state\":null,\"head\":{\"sha\":null}}' ;;\n\ + nohead) pr='{\"state\":\"open\",\"head\":{\"sha\":null}}' ;;\n\ + nostate) pr='{\"state\":null,\"head\":{\"sha\":\"aaa\"}}' ;;\n\ + *) pr=$(printf '{\"state\":\"%s\",\"head\":{\"sha\":\"%s\"},\"base\":{\"sha\":\"main-tip\"}}' \"$STUB_STATE\" \"$STUB_HEAD\") ;;\n\ +esac\n\ +printf '%s' \"$pr\" | jq -r \"$q\"\n"; + let bin = dir.path().join("bin"); + std::fs::create_dir(&bin).expect("bin dir"); + if !matches!(read, PrRead::NoGh) { + rocm_core::fake_executable::write_fake_executable(&bin.join("gh"), stub) + .expect("plant gh stub"); + } + // The stub and `jq` come from the real PATH; the no-`gh` case gets a PATH + // holding neither, and `bash` is named by path so the spawn still works. + let path = if matches!(read, PrRead::NoGh) { + bin.display().to_string() + } else { + format!( + "{}:{}", + bin.display(), + std::env::var("PATH").unwrap_or_default() + ) + }; + let mut cmd = std::process::Command::new("/bin/bash"); + cmd.arg("-e") + .arg(&script) + .env("PATH", path) + .env("GITHUB_RUN_ID", "42") + .env("GITHUB_OUTPUT", &output) + .env("GITHUB_STEP_SUMMARY", &summary); + for key in step_env(guard).keys() { + let value = match key.as_str() { + "GH_TOKEN" => "token", + "GH_REPO" => "owner/repo", + "PR_NUMBER" => "7", + "EVENT_HEAD_SHA" => "aaa", + other => panic!("unexpected guard env key `{other}`"), + }; + cmd.env(key, value); + } + match read { + PrRead::Pr(state, head) => { + cmd.env("STUB_STATE", state).env("STUB_HEAD", head); + } + PrRead::Fails => { + cmd.env("STUB_MODE", "fails"); + } + PrRead::Empty => { + cmd.env("STUB_MODE", "empty"); + } + PrRead::Nulls => { + cmd.env("STUB_MODE", "nulls"); + } + PrRead::NoHead => { + cmd.env("STUB_MODE", "nohead"); + } + PrRead::NoState => { + cmd.env("STUB_MODE", "nostate"); + } + PrRead::NoGh => {} + } + let out = cmd.output().expect("running the guard"); + let mut log = String::from_utf8_lossy(&out.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&out.stderr)); + ( + out.status.code().expect("guard exited with a status code"), + std::fs::read_to_string(&output).expect("read output"), + log, + std::fs::read_to_string(&summary).expect("read summary"), + ) + } + + /// `false` (after a loud note) when `jq` is missing off CI; panics on CI, + /// where a skip would read as a pass. + #[cfg(unix)] + fn jq_available_or_skip(test: &str) -> bool { + let has_jq = std::process::Command::new("jq") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()); + if !has_jq { + assert!( + std::env::var_os("CI").is_none(), + "`jq` is not on PATH, so `{test}` cannot run; install it on this runner" + ); + eprintln!("skipping {test}: no `jq` on PATH"); + } + has_jq + } + + #[test] + #[cfg(unix)] + fn stale_guard_flags_a_closed_or_superseded_pr_and_never_fails() { + if !jq_available_or_skip("stale_guard_flags_a_closed_or_superseded_pr_and_never_fails") { + return; + } + // Open and current: nothing is flagged. + let (code, out, _, summary) = run_stale_guard(PrRead::Pr("open", "aaa")); + assert_eq!((code, out.as_str(), summary.as_str()), (0, "", "")); + // Closed or merged: flagged, and noted in the step summary. + let (code, out, _, summary) = run_stale_guard(PrRead::Pr("closed", "aaa")); + assert_eq!((code, out.trim()), (0, "stale=true")); + assert!(summary.contains("is closed"), "summary: {summary}"); + // Head moved on: flagged, naming both heads. + let (code, out, log, summary) = run_stale_guard(PrRead::Pr("open", "bbb")); + assert_eq!((code, out.trim()), (0, "stale=true")); + assert!( + log.contains("aaa") && log.contains("bbb") && summary.contains("bbb"), + "log: {log}, summary: {summary}" + ); + // No `gh`: said so, and the job runs. + let (code, out, log, _) = run_stale_guard(PrRead::NoGh); + assert_eq!((code, out.as_str()), (0, ""), "{log}"); + assert!(log.contains("gh is not installed"), "log: {log}"); + // The PR cannot be read, or is read with a field missing: fail open, run + // the job. The stub rejects every call but `gh api`, so the guard + // cancelling anything would fail these. + for read in [ + PrRead::Fails, + PrRead::Empty, + PrRead::Nulls, + PrRead::NoHead, + PrRead::NoState, + ] { + let (code, out, log, _) = run_stale_guard(read); + assert_eq!( + (code, out.as_str()), + (0, ""), + "an unreadable PR, or no `gh`, must not flag the run stale: {log}" + ); + } + } + /// A job's `steps:` items as `(name, whole step)` pairs, in file order; /// `None` when the job has no `steps:` list. `block` is what `job_block` /// returns. @@ -1889,10 +2477,19 @@ jobs: for (workflow, text, report_job, step) in &steps { let (workflow, report_job) = (*workflow, *report_job); let block = job_block(text, report_job); - let lanes = self_hosted_e2e_jobs(text); - let expected: Vec = lanes + let mut lanes = self_hosted_e2e_jobs(text); + // A pull request's queue gate runs on a GitHub-hosted runner, but it + // is a lane all the same: it uploads a report, and can die without. + lanes.extend( + job_ids(text) + .into_iter() + .filter(|job| job_block(text, job).contains("gh workflow run queue-pr-run.yml")) + .map(|job| (job, String::from("the queue"))), + ); + let expected: Vec = job_ids(text) .iter() - .map(|(job, _)| format!("{job}={}", uploaded_report_artifact(workflow, text, job))) + .filter(|job| lanes.iter().any(|(lane, _)| lane == *job)) + .map(|job| format!("{job}={}", uploaded_report_artifact(workflow, text, job))) .collect(); // Read from the step itself, so a LANE_REPORTS left elsewhere in the // job after the step is deleted cannot satisfy this. @@ -1966,6 +2563,17 @@ jobs: step: &str, results: &[(&str, &str)], uploaded: Option<&[&str]>, + ) -> (i32, String, String) { + run_report_check_with_stale(step, results, &[], uploaded) + } + + /// As `run_report_check`, with `stale` naming the jobs whose `stale` output is `true`. + #[cfg(unix)] + fn run_report_check_with_stale( + step: &str, + results: &[(&str, &str)], + stale: &[&str], + uploaded: Option<&[&str]>, ) -> (i32, String, String) { let dir = tempfile::tempdir().expect("temp dir"); let script = dir.path().join("check.sh"); @@ -1986,7 +2594,24 @@ jobs: let needs: serde_json::Map = results .iter() - .map(|(job, result)| ((*job).to_owned(), serde_json::json!({ "result": result }))) + .map(|(job, result)| { + // "success+skipped": a lane that skipped itself on purpose says + // so in its job outputs; a stale lane says `stale` in its own. + let (result, skipped) = result + .split_once('+') + .map_or((*result, ""), |(r, o)| (r, o)); + let mut outputs = serde_json::Map::new(); + if !skipped.is_empty() { + outputs.insert("skipped".to_owned(), "true".into()); + } + if stale.contains(job) { + outputs.insert("stale".to_owned(), "true".into()); + } + ( + (*job).to_owned(), + serde_json::json!({ "result": result, "outputs": outputs }), + ) + }) .collect(); let path = format!( "{}:{}", @@ -2019,24 +2644,47 @@ jobs: ) } + #[test] + #[cfg(unix)] + fn missing_report_check_ignores_a_lane_that_skipped_itself_as_stale() { + if !jq_available_or_skip("missing_report_check_ignores_a_lane_that_skipped_itself_as_stale") + { + return; + } + let (.., step) = report_check_steps() + .into_iter() + .find(|(workflow, ..)| *workflow == "e2e-selfhosted.yml") + .expect("the per-PR workflow has a report check"); + let results = [ + ("e2e-gpu", "success"), + // The queue gate is a lane of this check too: skipped here, as for a + // pull request that gets no queue run. + ("e2e-gpu-queue", "skipped"), + ("e2e-gpu-strix-ubuntu", "skipped"), + ("e2e-gpu-strix-windows", "skipped"), + ("e2e-wsl", "skipped"), + ]; + let (code, ..) = run_report_check_with_stale(&step, &results, &[], Some(&[])); + assert_eq!(code, 1, "a lane that ran without a report is still flagged"); + let (code, log, summary) = + run_report_check_with_stale(&step, &results, &["e2e-gpu"], Some(&[])); + assert_eq!( + code, 0, + "a stale-skipped lane is not a missing report: {log}" + ); + assert!( + summary.contains("`e2e-gpu` skipped itself as stale"), + "the lane's absence from the grid is said in the summary: {summary}" + ); + } + #[test] #[cfg(unix)] fn missing_report_check_flags_only_lanes_that_ran_without_a_report() { // The contract test above pins the check's inputs; this runs its logic, // so a check that never fails (or always does) cannot pass for one. - let has_jq = std::process::Command::new("jq") - .arg("--version") - .output() - .is_ok_and(|out| out.status.success()); - if !has_jq { - // A skip reads as a pass, so on CI a missing `jq` must fail loudly - // rather than leave the check's logic silently untested. - assert!( - std::env::var_os("CI").is_none(), - "`jq` is not on PATH, so the missing-report check's logic cannot be \ - exercised; install it on this runner" - ); - eprintln!("skipping: no `jq` on PATH"); + if !jq_available_or_skip("missing_report_check_flags_only_lanes_that_ran_without_a_report") + { return; } let (.., step) = report_check_steps() @@ -2101,6 +2749,16 @@ jobs: "a missing report fails whatever the needs result:\n{log}" ); + // A gate that skipped itself for the nightly window ends green with no + // report, and is not missing; the same lane without that output is. + let mut self_skipped = all("success"); + self_skipped[0].1 = "success+skipped"; + let (code, log, _) = run_report_check(&step, &self_skipped, Some(&without_victim)); + assert_eq!( + code, 0, + "a lane that skipped itself on purpose is not missing:\n{log}" + ); + let mut skipped = all("success"); skipped[0].1 = "skipped"; let (code, log, _) = run_report_check(&step, &skipped, Some(&without_victim)); @@ -2294,15 +2952,41 @@ jobs: /// wrapped into a block scalar (`name: >-`) is found exactly like a one-line /// one, and a `- name: GPU preflight` line in a comment or some step's /// script is not a step. A job with no `steps:` list contributes nothing. + /// Exception: `e2e-gpu`'s preflight comes back without its stale-run gate. fn gpu_preflight_steps(text: &str) -> Vec { job_ids(text) .into_iter() - .flat_map(|job| job_steps(job_block(text, &job)).unwrap_or_default()) - .filter(|(name, _)| name.starts_with("GPU preflight")) - .map(|(_, step)| step) + .flat_map(|job| { + job_steps(job_block(text, &job)) + .unwrap_or_default() + .into_iter() + .map(move |(name, step)| (job.clone(), name, step)) + }) + .filter(|(_, name, _)| name.starts_with("GPU preflight")) + .map(|(job, _, step)| { + // Only `e2e-gpu` carries the stale-run guard, which gates every + // later step of that job on one `if:`, the preflight included. + // That line is dropped from that one job's preflight so the + // shape checks keep pinning it as a bare delegation, while + // `mi300x_lane_skips_itself_when_its_run_is_stale` requires the gate + // itself on every step of the lane. Any other lane that grows + // an `if:` on its preflight still fails those checks. + if job != "e2e-gpu" { + return step; + } + let mut kept = String::new(); + for line in step.lines().filter(|l| l.trim() != STALE_RUN_GATE) { + kept.push_str(line); + kept.push('\n'); + } + kept + }) .collect() } + /// The `if:` line the MI300X lane's stale-run guard puts on its later steps. + const STALE_RUN_GATE: &str = "if: steps.stale.outputs.stale != 'true'"; + /// The whole step whose `- name:` line is `lines[at]`: that line and every /// line below it until a non-blank one is indented no deeper than it. /// @@ -2872,8 +3556,8 @@ jobs: } } assert_eq!( - delegating, 7, - "expected the seven native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the four native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2926,7 +3610,8 @@ jobs: "{name} is no longer just a named, floor-carrying delegation: a \ preflight that also sets `if:` or `continue-on-error:` is a gate \ the job can skip or survive, which is the one failure mode a \ - required gate has" + required gate has (the one exception, `e2e-gpu`'s stale-run \ + gate, is removed by `gpu_preflight_steps`)" ); assert_eq!( step_env(step).keys().collect::>(), @@ -2941,8 +3626,8 @@ jobs: } assert_eq!( bodies.len(), - 7, - "expected seven native Linux preflight bodies to compare, found {} — \ + 4, + "expected four native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4986,8 @@ fi } } assert_eq!( - delegating, 7, - "expected the 7 native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the 4 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( @@ -4399,14 +5084,137 @@ fi } } + /// The nightly report carries the shared queue's GPUs into the matrix, and + /// so into the wiki and the release-readiness verdict, from the queue's own + /// results: the newest finished run on `main`, found by its artifact's name + /// (no workflow may name or wait on the queue's, see + /// `assert_queue_workflow_holds_no_credential`). Neither step may fail the + /// job: a missing or late run is said so at the top of the page. #[test] - fn nightly_publishes_the_consolidated_matrix_to_the_wiki() { + fn nightly_report_imports_the_latest_queue_run_without_depending_on_it() { let nightly = read_workflow("nightly.yml"); - let report = normalized_whitespace(job_block(&nightly, "e2e-report-nightly")); - assert!( - report.contains("--html-out consolidated/index.html > consolidated/support-matrix.md"), - "the consolidated report job must redirect the markdown matrix into \ - consolidated/support-matrix.md for the wiki job" + let job = job_block(&nightly, "e2e-report-nightly"); + let steps = job_steps(job).expect("the report job has steps"); + let position = |prefix: &str| { + steps + .iter() + .position(|(name, _)| name.starts_with(prefix)) + .unwrap_or_else(|| panic!("e2e-report-nightly has a `{prefix}` step")) + }; + let (fetch, add, build) = ( + position("Fetch the latest queue results"), + position("Add the queue lanes to the report"), + position("Build consolidated report"), + ); + assert!( + fetch < add && add < build, + "the queue lanes must be fetched, then put among the reports, before the report is built" + ); + for at in [fetch, add] { + assert_eq!( + step_key(&steps[at].1, "continue-on-error"), + "true", + "`{}` must not be able to fail the nightly report: a missing or late queue run \ + is shown as missing, not as a red report", + steps[at].0 + ); + } + + let fetch = normalized_whitespace(&steps[fetch].1); + for needed in [ + // By artifact name, from a run on this repository's main: the + // workflow that makes it may not be named here. + "actions/artifacts?name=queue-run-results", + ".workflow_run.head_branch == \"main\"", + ".workflow_run.head_repository_id == $repo", + ".expired == false", + // Today's run: not yesterday's, whatever its conclusion was. + "date -u -d '24 hours ago'", + ".created_at >= $since", + "gh run download \"$candidate\"", + "--name queue-run-results", + // A run that started a day ago is not tonight's, whenever it ended. + "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", + // Only a finished run: a run still going has a partial log. + "[ \"${run_info%%$'\\t'*}\" = \"completed\" ]", + // The log says which branch the run tested and which lanes block; + // the run may have several jobs, so every one is scanned. + "actions/runs/${candidate}/jobs?per_page=100", + "/logs\" > queue-results/job.log.part", + ] { + assert!( + fetch.contains(needed), + "the queue fetch step must contain `{needed}`:\n{fetch}" + ); + } + assert!( + !fetch.contains("conclusion"), + "a red queue run is still a run: its artifact is uploaded whatever it concluded" + ); + // No credential beyond the job's own read-only token. + let fetch_step = &steps[position("Fetch the latest queue results")].1; + let env = step_env(fetch_step); + assert_eq!( + env.get("GH_TOKEN").map(String::as_str), + Some("${{ github.token }}"), + "{env:?}" + ); + assert_eq!(env.len(), 1, "{env:?}"); + assert!(!fetch_step.contains("secrets."), "{fetch_step}"); + let permissions = job_mapping(job, "permissions"); + assert_eq!( + permissions.get("actions").map(String::as_str), + Some("read"), + "listing and downloading the queue's artifact needs actions: read" + ); + assert_eq!( + permissions.get("contents").map(String::as_str), + Some("read") + ); + assert_eq!(permissions.len(), 2, "{permissions:?}"); + + // Both outcomes go through the importer, which writes the status section. + let add = normalized_whitespace(&steps[position("Add the queue lanes to the report")].1); + for needed in [ + "cargo xtask e2e-queue-import", + "--queue-dir queue-results/results", + "--absent", + "--artifacts-dir e2e-artifacts", + "--section-out queue-results/queue-lanes.md", + // An import that fails leaves no half of one behind, and says so. + "rm -rf e2e-artifacts/e2e-queue-*", + "--absent \"the queue lanes could not be imported\"", + ] { + assert!( + add.contains(needed), + "the queue import step must contain `{needed}`:\n{add}" + ); + } + // The section leads the page; the matrix keeps its own first table. + let build = normalized_whitespace(&steps[build].1); + assert!( + build.contains( + "cat queue-results/queue-lanes.md consolidated/support-matrix.md > \ + \"$RUNNER_TEMP/with-queue.md\"" + ), + "the queue section must be put ahead of the matrix:\n{build}" + ); + // A queue run is never something the report job waits for. + let needs = job_scalar(job, "needs"); + assert!( + !needs.contains("queue"), + "e2e-report-nightly must not depend on the queue's workflows: {needs}" + ); + } + + #[test] + fn nightly_publishes_the_consolidated_matrix_to_the_wiki() { + let nightly = read_workflow("nightly.yml"); + let report = normalized_whitespace(job_block(&nightly, "e2e-report-nightly")); + assert!( + report.contains("--html-out consolidated/index.html > consolidated/support-matrix.md"), + "the consolidated report job must redirect the markdown matrix into \ + consolidated/support-matrix.md for the wiki job" ); assert!( report.contains("cat consolidated/support-matrix.md >> \"$GITHUB_STEP_SUMMARY\""), @@ -4630,6 +5438,7 @@ fi failing_gates: vec!["E2E (report unreadable)".to_string()], e2e_fail_total: None, build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); @@ -4649,6 +5458,7 @@ fi failing_gates: vec![], e2e_fail_total: Some(0), build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); @@ -4852,11 +5662,8 @@ fi } for lane in [ "e2e-gpu-nightly", - "e2e-gpu-nightly-rad3", - "e2e-gpu-nightly-mi350p", "e2e-gpu-nightly-strix", "e2e-gpu-nightly-strix-windows", - "e2e-gpu-nightly-w7900", "e2e-wsl-nightly", ] { assert!( @@ -5073,14 +5880,16 @@ fi let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) .expect("read hardware testing docs"); - let documented: Vec> = - markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |") - .into_iter() - .filter(|row| { - row.get(1) - .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") - }) - .collect(); + let documented: Vec> = markdown_table_rows( + &docs, + "| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) |", + ) + .into_iter() + .filter(|row| { + row.get(1) + .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") + }) + .collect(); let documented_jobs: Vec = documented.iter().map(|row| row[0].clone()).collect(); let declared_jobs: Vec = lanes.iter().map(|(job, _)| format!("`{job}`")).collect(); @@ -5108,6 +5917,15 @@ fi flattened_list_items(runs_on), "the documented runner labels for `{job}` must match its actual runs-on" ); + let declared = job_mapping(job_block(&self_hosted, job), "env") + .get(e2e_report::EXPECT_GPU_ENV) + .map(|value| format!("`{value}`")); + assert_eq!( + row.get(4), + declared.as_ref(), + "the GPU declared cell for `{job}` must match its job-level {}", + e2e_report::EXPECT_GPU_ENV + ); } // Derived from the same scan `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` @@ -5734,15 +6552,21 @@ fi ); } - /// The hermetic comfyui-01/02/03 scenarios (no GPU, no network, no - /// `e2e-test-hooks` dependency, Windows-portable since #637) must stay - /// explicitly selected into this otherwise-`@lifecycle`-only Windows lane - /// (#689) — it is required CI, unlike the self-hosted lane that otherwise - /// runs them. This checks the whole step (not just the `run: |` body, since - /// the selection lives in `env:`), so dropping the env var — or narrowing - /// its id list — fails here rather than silently reopening the coverage gap. + /// The hermetic non-`@lifecycle` scenarios this otherwise-`@lifecycle`-only + /// Windows lane selects through `E2E_ALSO_RUN_IDS` (#689): comfyui-01/02/03 + /// (Windows-portable since #637), the native-Windows vLLM refusal + /// serve-33/34/35, and the side-by-side runtime scenarios + /// runtime-lifecycle-19/20/21. None needs a GPU, the network or + /// `e2e-test-hooks`, and this is required CI, unlike the self-hosted lane + /// that otherwise runs them. This checks the whole step (not just the + /// `run: |` body, since the selection lives in `env:`) and pins the id list + /// exactly, so dropping the env var, narrowing the list or growing it + /// without updating this test fails here. It also holds the PowerShell + /// copy of the list in docs/testing.md equal to the ci.yml one; the + /// "19/20/21"-style shorthand in docs/testing.md's prose and in + /// tests/e2e-cucumber/README.md is not checked. #[test] - fn ci_windows_lifecycle_lane_also_selects_the_hermetic_comfyui_and_vllm_refusal_scenarios() { + fn ci_windows_lifecycle_lane_also_selects_exactly_the_hermetic_non_lifecycle_scenarios() { let ci = read_workflow("ci.yml"); let windows = job_block(&ci, "windows-release"); let lines: Vec<&str> = windows.lines().collect(); @@ -5761,24 +6585,62 @@ fi also_run_ids.len(), 1, "the Windows lifecycle lane must set env.E2E_ALSO_RUN_IDS exactly \ - once to additionally select the hermetic comfyui and vLLM-refusal scenarios, so \ - they get required-CI Windows coverage (#689):\n{step}" - ); - for id in [ - "comfyui-uv-install-failure-names-log", - "comfyui-uv-install-progress-streamed", - "comfyui-ambiguous-runtime-actionable", - // The native-Windows vLLM refusal (serve-33/34/35) is - // `@requires-os:windows` and hermetic, so this is its required lane. - "serve-vllm-refused-on-native-windows", - "serve-vllm-install-refused-on-native-windows", - "serve-vllm-install-with-runtime-refused-on-native-windows", - ] { - assert!( - also_run_ids[0].contains(id), - "the Windows lifecycle lane must select {id} via env.E2E_ALSO_RUN_IDS:\n{step}" - ); - } + once to additionally select the hermetic comfyui, vLLM-refusal and \ + side-by-side runtime scenarios, so they get required-CI Windows \ + coverage (#689):\n{step}" + ); + let selected: Vec<&str> = also_run_ids[0] + .trim_matches('"') + .split(',') + .map(str::trim) + .collect(); + assert_eq!( + selected, + [ + "comfyui-uv-install-failure-names-log", + "comfyui-uv-install-progress-streamed", + "comfyui-ambiguous-runtime-actionable", + // The native-Windows vLLM refusal (serve-33/34/35) is + // `@requires-os:windows` and hermetic, so this is its required lane. + "serve-vllm-refused-on-native-windows", + "serve-vllm-install-refused-on-native-windows", + "serve-vllm-install-with-runtime-refused-on-native-windows", + // runtime-lifecycle-19/20/21: planted side-by-side runtimes, + // hermetic, whose Windows paths only this lane exercises. + "runtime-lifecycle-side-by-side-across-packaging-change", + "runtime-lifecycle-side-by-side-uninstall-keeps-the-others", + "runtime-lifecycle-side-by-side-shared-name-refused", + ], + "the Windows lifecycle lane's env.E2E_ALSO_RUN_IDS must select exactly the \ + hermetic comfyui-01/02/03, serve-33/34/35 and runtime-lifecycle-19/20/21 \ + scenarios; update this list, docs/testing.md and \ + tests/e2e-cucumber/README.md together with ci.yml:\n{step}" + ); + + // docs/testing.md shows the same selection as a PowerShell line to run + // the lane locally; it must not drift from what CI selects. + let testing = std::fs::read_to_string(repo_root().join("docs/testing.md")) + .expect("docs/testing.md must be readable") + .replace("\r\n", "\n"); + let documented: Vec<&str> = testing + .lines() + .filter_map(|line| line.trim().strip_prefix("$env:E2E_ALSO_RUN_IDS = ")) + .collect(); + assert_eq!( + documented.len(), + 1, + "docs/testing.md must show the Windows lane's E2E_ALSO_RUN_IDS once" + ); + let documented: Vec<&str> = documented[0] + .trim_matches('"') + .split(',') + .map(str::trim) + .collect(); + assert_eq!( + documented, selected, + "docs/testing.md's $env:E2E_ALSO_RUN_IDS line must list exactly the ids \ + ci.yml's Windows lifecycle lane selects, in the same order" + ); } /// Every job in `ci.yml` as `(id, block)`, in file order. Unlike [`job_block`] @@ -7593,9 +8455,34 @@ jobs: "github.event_name != 'merge_group' && ", "app-dev-gpu", ), + // A pull request's queue lane: only on a `pull_request` from this + // repository, and never on `closed`, which skips `changes` and so it. + PinnedIf { + job: "e2e-gpu-queue", + condition: "needs.changes.result == 'success' \ + && needs.changes.outputs.serve == 'true' \ + && github.event_name == 'pull_request' \ + && github.event.action != 'closed' \ + && github.event.pull_request.head.repo.full_name \ + == github.repository \ + && github.actor != 'dependabot[bot]'" + .to_owned(), + matrix_lane: false, + }, off_pr_lane("e2e-gpu-strix-ubuntu", "strix-ubuntu"), lane("e2e-gpu-strix-windows", "", "strix-windows"), lane("e2e-wsl", "", "strix-wsl"), + // Skips on `closed` because the `stale` output of a skipped `e2e-gpu` + // is empty. (Its full condition is also pinned by + // `stale_runs_are_cancelled_by_a_hosted_job_that_runs_no_pr_code`.) + PinnedIf { + job: "cancel-stale-run", + condition: "always() && github.event_name == 'pull_request' \ + && github.event.pull_request.head.repo.full_name == github.repository \ + && needs.e2e-gpu.outputs.stale == 'true'" + .to_owned(), + matrix_lane: false, + }, // Skips because the `serve` output of a skipped `changes` is empty. PinnedIf { job: "e2e-report", @@ -9219,7 +10106,12 @@ permissions: } /// The workflows that may run on the queue-report runner. - const QUEUE_WORKFLOWS: [&str; 2] = ["nightly-queue.yml", "queue-run.yml"]; + const QUEUE_WORKFLOWS: [&str; 3] = ["nightly-queue.yml", "queue-pr-run.yml", "queue-run.yml"]; + + /// The workflows that may dispatch `queue-pr-run.yml`, on a GitHub-hosted + /// runner: a pull request's gate job. Naming it is all they may do of the + /// queue's; the runner's label and the other two workflows stay refused. + const QUEUE_PR_RUN_CALLERS: [&str; 1] = ["e2e-selfhosted.yml"]; #[test] fn queue_nightly_publisher_holds_no_credential() { @@ -9230,6 +10122,804 @@ permissions: ); } + #[test] + fn queue_pr_run_holds_no_credential() { + // The branch and the caller's run reach it as inputs: the branch only + // through `env:` into the publisher, the caller only into the run's + // title. + assert_queue_workflow_holds_no_credential( + "queue-pr-run.yml", + &[ + "${{inputs.branch}}", + "${{inputs.caller}}", + "${{runner.temp}}", + ], + &["workflow_dispatch:"], + ); + } + + /// Anyone who can dispatch names the branch, not only the gate, so the + /// workflow on main checks it before the publisher sees it: a plain branch + /// name, which cannot start with a dash and be read as an option. + #[test] + fn queue_pr_run_checks_the_branch_name_before_it_is_published() { + let text = read_workflow("queue-pr-run.yml"); + let check = text + .find(" - name: Check the branch name") + .expect("queue-pr-run.yml checks the branch name"); + let publish = text + .find(" - name: Run on the queue and publish") + .expect("queue-pr-run.yml publishes the run"); + assert!(check < publish, "the check must come first"); + assert!( + text.contains("=~ ^[A-Za-z0-9_][A-Za-z0-9._/-]{0,199}$ ]]"), + "the check must accept only a plain branch name" + ); + } + + /// What the pull request's gate job may and may not do: it holds no queue + /// credential, dispatches the workflow as it is on main, and publishes only + /// the queue run's release report as the MI300X one. + #[test] + fn the_pull_request_gate_dispatches_the_workflow_on_main_and_nothing_else() { + let text = read_workflow("e2e-selfhosted.yml"); + let gate = job_block(&text, "e2e-gpu-queue"); + // GitHub-hosted: it needs no runner of the queue's, and no label of it. + assert_eq!(runs_on_values(gate), ["ubuntu-24.04"]); + // The one permission it needs, to dispatch: no contents, no token to push. + assert_eq!( + gate.matches("permissions:").count(), + 1, + "the gate declares its permissions once" + ); + // The whole mapping, so a second permission written beside it fails here: + // every line indented under `permissions:` must be exactly this one. + let permissions: Vec<&str> = gate + .lines() + .skip_while(|line| *line != " permissions:") + .skip(1) + .take_while(|line| line.starts_with(" ")) + .collect(); + assert_eq!( + permissions, + [" actions: write"], + "the gate's only permission is `actions: write`" + ); + // A job whose `if` stays true through a cancellation is not cancelled, + // and a superseded gate would poll its queue run for hours. + let condition = job_level_if(&uncommented_job_block(&text, "e2e-gpu-queue")); + assert!( + !condition.contains("always()"), + "the gate's `if` must not hold `always()`: {condition}" + ); + assert!( + gate.contains("if: failure() || cancelled()") && gate.contains("gh run cancel"), + "the gate must cancel the queue run it started when it fails or is cancelled" + ); + // From main, never from the pull request's own copy of the workflow: the + // runner runs only the file as it is on main. + assert!( + gate.contains("gh workflow run queue-pr-run.yml --ref main -f \"branch=${HEAD_REF}\""), + "the gate must dispatch queue-pr-run.yml on main with the PR's branch" + ); + // The nightly window is a skip, not a failure: only that refusal ends + // green, and it says so in an output the report check reads. + assert!( + gate.contains("== nightly:* ]]") + && gate.contains("skipped=true") + && gate.contains("Re-run this job once the window has passed"), + "a nightly-window refusal must end the gate green with a skip note" + ); + assert!( + gate.contains("outputs:\n skipped: ${{ steps.queue.outputs.skipped }}"), + "the gate must expose `skipped` for the missing-report check" + ); + // Only a pull request from this repository: a fork cannot dispatch. + for needed in [ + "github.event_name == 'pull_request'", + "github.event.action != 'closed'", + "github.event.pull_request.head.repo.full_name == github.repository", + // Dependabot's token is read-only whatever `permissions:` says. + "github.actor != 'dependabot[bot]'", + ] { + assert!( + gate.contains(needed), + "the gate must only run when {needed}" + ); + } + // The PR author's branch name reaches the script through the environment, + // never as text of it, and is checked before it is used. + assert!( + gate.contains("HEAD_REF: ${{ github.event.pull_request.head.ref }}"), + "the branch name must come through env" + ); + // Only the first step's script: stop at the next step, whose `with:` block + // may legitimately hold an expression. + let script: Vec<&str> = gate + .lines() + .skip_while(|line| !line.trim_start().starts_with("run: |")) + .enumerate() + .take_while(|(i, line)| *i == 0 || !line.starts_with(" - name:")) + .map(|(_, line)| line) + .collect(); + assert!( + !script.iter().any(|line| line.contains("${{")), + "no expression may sit inside the gate's script" + ); + assert!( + gate.contains("=~ ^[A-Za-z0-9_][A-Za-z0-9._/-]{0,199}$"), + "the branch name must be checked before it is passed on" + ); + // Its one report, the MI300X cell the queue's run stands for. + assert_eq!( + uploaded_report_artifact("e2e-selfhosted.yml", &text, "e2e-gpu-queue"), + "e2e-gpu-report" + ); + assert_eq!( + uploaded_report_artifact("e2e-selfhosted.yml", &text, "e2e-gpu"), + "e2e-gpu-selfhosted-report", + "the self-hosted lane must not take the queue's report name" + ); + // And no other job dispatches or names the queue's workflow. + for job in job_ids(&text) { + if job != "e2e-gpu-queue" { + assert!( + !without_comments(job_block(&text, &job)).contains("queue-pr-run"), + "job `{job}` names queue-pr-run.yml: only the gate may" + ); + } + } + } + + /// The queue side's half of the gate's contract, as the gate reads it: the + /// first line of `refused.txt` starts with this when the queue refused for + /// the nightly's window, and the MI300X lane's release report sits at this + /// path of the `queue-pr-run-results` artifact. The queue side is not in this + /// repository: these are the gate's half, shared by the script check, the + /// mock runs and the doc check below so none can drift alone. + const QUEUE_NIGHTLY_REFUSAL: &str = "nightly:"; + const QUEUE_REPORT_PATH: &str = "mi300x/release/results/report.json"; + + /// One run of the gate's script against a mock `gh`: what to answer and what + /// the queue run published. + #[cfg(unix)] + struct Gate<'a> { + /// `queue-pr-run.yml` on main: "present", "absent" (HTTP 404), else an error. + main: &'a str, + /// The same file on the pull request's branch: "present" or "absent". + head: &'a str, + head_ref: &'a str, + /// Calls of `gh run list` that fail before one answers. + list_fail: u32, + /// Whether the dispatched run is in the list at all. + listed: bool, + /// Calls of `gh run view --json status` that fail, then that say running. + status_fail: u32, + status_running: u32, + /// Calls of `gh run view --json conclusion` that fail before one answers. + conclusion_fail: u32, + conclusion: &'a str, + /// What the artifact holds, `(path, content)`; `None` is no artifact. + results: Option>, + /// The repository variable `QUEUE_PR_GATE`. + switch: &'a str, + /// Run the step that cancels the queue run instead of the main one, with + /// the files the main step leaves in `RUNNER_TEMP`: whether it got as far + /// as dispatching, and the run id it found, if it did. + cancel_step: bool, + dispatched: bool, + run_id_file: Option<&'a str>, + } + + #[cfg(unix)] + impl Default for Gate<'_> { + fn default() -> Self { + Gate { + main: "present", + head: "present", + head_ref: "ci/some-branch", + list_fail: 0, + listed: true, + status_fail: 0, + status_running: 0, + conclusion_fail: 0, + conclusion: "success", + results: Some(vec![(QUEUE_REPORT_PATH, "{}")]), + switch: "true", + cancel_step: false, + dispatched: true, + run_id_file: None, + } + } + } + + #[cfg(unix)] + struct GateRun { + code: i32, + log: String, + summary: String, + output: String, + /// Every `gh` and `sleep` call, one per line, in order. + calls: Vec, + /// Files copied to `report/`. + report: Vec, + } + + #[cfg(unix)] + impl GateRun { + fn calls_starting(&self, prefix: &str) -> usize { + self.calls.iter().filter(|c| c.starts_with(prefix)).count() + } + } + + /// Run the gate's first step as `bash -e`, as a `run:` step does, in an empty + /// directory with a mock `gh` and a `sleep` that does not sleep. + #[cfg(unix)] + fn run_gate_script(gate: &Gate) -> GateRun { + let text = read_workflow("e2e-selfhosted.yml"); + let block = job_block(&text, "e2e-gpu-queue"); + let block = if gate.cancel_step { + block + .split_once(" - name: Cancel the queue run") + .expect("the gate has a cancel step") + .1 + } else { + block + }; + let script = dedent(&run_block(block).expect("the step has a `run: |` block")) + .expect("the step's script is not empty"); + let dir = tempfile::tempdir().expect("temp dir"); + let bin = dir.path().join("bin"); + let state = dir.path().join("state"); + let work = dir.path().join("work"); + let results = dir.path().join("results"); + for d in [&bin, &state, &work] { + std::fs::create_dir_all(d).expect("make dir"); + } + std::fs::write(dir.path().join("gate.sh"), script).expect("write script"); + if gate.dispatched { + std::fs::write(dir.path().join("queue-dispatched"), "").expect("write marker"); + } + if let Some(id) = gate.run_id_file { + std::fs::write(dir.path().join("queue-run-id"), id).expect("write run id"); + } + for f in ["output", "summary"] { + std::fs::write(dir.path().join(f), "").expect("write file"); + } + // Titles as the queue run's are: the gate looks for its own `caller`. + let own = if gate.listed { + r#",{"databaseId":222,"displayTitle":"PR run of b; caller 7-1; at x"}"# + } else { + "" + }; + std::fs::write( + state.join("listing.json"), + format!( + r#"[{{"databaseId":111,"displayTitle":"PR run of a; caller 99-1; at x"}},{{"databaseId":112,"displayTitle":"PR run of c; caller 17-1; at x"}},{{"databaseId":113,"displayTitle":"PR run of d; caller 7-10; at x"}}{own}]"# + ), + ) + .expect("write listing"); + if let Some(files) = &gate.results { + std::fs::create_dir_all(&results).expect("make results"); + for (path, content) in files { + let target = results.join(path); + std::fs::create_dir_all(target.parent().expect("a parent")).expect("make dir"); + std::fs::write(target, content).expect("write result"); + } + } + // `bump KEY` counts the calls of one kind; the answers below follow it. + let gh = r#"#!/bin/sh +echo "$*" >> "$STATE/calls" +bump() { n=$(cat "$STATE/$1" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$STATE/$1"; echo "$n"; } +case "$1" in +api) + case "$2" in + *"?ref=main") + case "$MAIN" in + present) echo queue-pr-run.yml ;; + absent) echo "gh: Not Found (HTTP 404)" >&2; exit 1 ;; + *) echo "HTTP 502: bad gateway" >&2; exit 1 ;; + esac ;; + *) + if [ "$HEAD" = present ]; then echo queue-pr-run.yml; else echo "gh: Not Found (HTTP 404)" >&2; exit 1; fi ;; + esac ;; +workflow) exit 0 ;; +run) + case "$2" in + list) + n=$(bump list) + if [ "$n" -le "$LIST_FAIL" ]; then echo "HTTP 502" >&2; exit 1; fi + cat "$STATE/listing.json" ;; + view) + case "$5" in + status) + n=$(bump status) + if [ "$n" -le "$STATUS_FAIL" ]; then echo "HTTP 502" >&2; exit 1; fi + if [ "$n" -le $((STATUS_FAIL + STATUS_RUNNING)) ]; then s=in_progress; else s=completed; fi + printf '{"status":"%s"}' "$s" | jq -r "$7" ;; + conclusion) + n=$(bump conclusion) + if [ "$n" -le "$CONCLUSION_FAIL" ]; then echo "HTTP 502" >&2; exit 1; fi + printf '{"conclusion":"%s"}' "$CONCLUSION" | jq -r "$7" ;; + esac ;; + download) + if [ -d "$RESULTS" ]; then cp -R "$RESULTS/." "$7"; else echo "no artifact" >&2; exit 1; fi ;; + esac ;; +esac +"#; + rocm_core::fake_executable::write_fake_executable(&bin.join("gh"), gh).expect("plant gh"); + rocm_core::fake_executable::write_fake_executable( + &bin.join("sleep"), + "#!/bin/sh\necho \"sleep $*\" >> \"$STATE/calls\"\n", + ) + .expect("plant sleep"); + + let path = format!( + "{}:{}", + bin.display(), + std::env::var("PATH").unwrap_or_default() + ); + let out = std::process::Command::new("bash") + .arg("-e") + .arg(dir.path().join("gate.sh")) + .current_dir(&work) + .env("PATH", path) + .env("STATE", &state) + .env("MAIN", gate.main) + .env("HEAD", gate.head) + .env("LIST_FAIL", gate.list_fail.to_string()) + .env("STATUS_FAIL", gate.status_fail.to_string()) + .env("STATUS_RUNNING", gate.status_running.to_string()) + .env("CONCLUSION_FAIL", gate.conclusion_fail.to_string()) + .env("CONCLUSION", gate.conclusion) + .env("RESULTS", &results) + .env("GH_REPO", "owner/repo") + .env("HEAD_REF", gate.head_ref) + .env("QUEUE_PR_GATE", gate.switch) + .env("GITHUB_RUN_ID", "7") + .env("GITHUB_RUN_ATTEMPT", "1") + .env("GITHUB_OUTPUT", dir.path().join("output")) + .env("GITHUB_STEP_SUMMARY", dir.path().join("summary")) + .env("RUNNER_TEMP", dir.path()) + .output() + .expect("running the gate script"); + let mut log = String::from_utf8_lossy(&out.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&out.stderr)); + let read = |p: std::path::PathBuf| std::fs::read_to_string(p).unwrap_or_default(); + let mut report: Vec = std::fs::read_dir(work.join("report")) + .map(|rd| { + rd.filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + report.sort(); + GateRun { + code: out + .status + .code() + .expect("the gate exited with a status code"), + log, + summary: read(dir.path().join("summary")), + output: read(dir.path().join("output")), + calls: read(state.join("calls")) + .lines() + .map(str::to_owned) + .collect(), + report, + } + } + + /// The mock runs read `jq`, as the gate does; on CI a missing one must fail + /// loudly, as for the missing-report check, rather than skip the tests. + #[cfg(unix)] + fn jq_available() -> bool { + let has_jq = std::process::Command::new("jq") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()); + if !has_jq { + assert!( + std::env::var_os("CI").is_none(), + "`jq` is not on PATH, so the gate script cannot be exercised; install it on this runner" + ); + eprintln!("skipping: no `jq` on PATH"); + } + has_jq + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_passes_a_green_run_and_publishes_its_release_report() { + if !jq_available() { + return; + } + let run = run_gate_script(&Gate { + results: Some(vec![ + (QUEUE_REPORT_PATH, "{}"), + ("mi300x/release/results/platform.json", "{}"), + // Another lane's and another channel's files are not this job's. + ("w7900/release/results/report.json", "{}"), + ("mi300x/nightly/results/report.json", "{}"), + ]), + ..Gate::default() + }); + assert_eq!(run.code, 0, "{}", run.log); + assert_eq!(run.report, ["platform.json", "report.json"], "{}", run.log); + assert!(!run.output.contains("skipped"), "{}", run.output); + // Dispatched from main with the branch and this job's caller, and the run + // watched is the one whose title carries that caller: not a neighbour's. + assert!( + run.calls.contains( + &"workflow run queue-pr-run.yml --ref main -f branch=ci/some-branch -f caller=7-1" + .to_owned() + ), + "{:?}", + run.calls + ); + assert!( + run.calls.iter().any(|c| c.starts_with("run view 222 ")), + "{:?}", + run.calls + ); + assert!( + !run.calls.iter().any(|c| c.starts_with("run view 111")), + "{:?}", + run.calls + ); + assert!( + run.calls.iter().any(|c| c.starts_with("run download 222 ")), + "{:?}", + run.calls + ); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_follows_the_runs_verdict_but_still_publishes_its_report() { + if !jq_available() { + return; + } + // A failing suite fails the gate, and its report is what the grid shows. + let failed = run_gate_script(&Gate { + conclusion: "failure", + ..Gate::default() + }); + assert_eq!(failed.code, 1, "{}", failed.log); + assert_eq!(failed.report, ["report.json"], "{}", failed.log); + // A run with no MI300X release report is red, whatever else it published. + let none = run_gate_script(&Gate { + results: Some(vec![("w7900/release/results/report.json", "{}")]), + ..Gate::default() + }); + assert_eq!(none.code, 1, "{}", none.log); + assert!(none.report.is_empty()); + assert!( + none.log.contains("left no MI300X release report"), + "{}", + none.log + ); + // So is a run that published no artifact at all, even a cancelled one. + let gone = run_gate_script(&Gate { + results: None, + conclusion: "cancelled", + ..Gate::default() + }); + assert_eq!(gone.code, 1, "{}", gone.log); + assert!(gone.log.contains("it cancelled"), "{}", gone.log); + assert!(!gone.output.contains("skipped"), "{}", gone.output); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_skips_only_for_the_nightly_window() { + if !jq_available() { + return; + } + let nightly = format!("{QUEUE_NIGHTLY_REFUSAL} the nightly's GPUs are reserved\n"); + let window = run_gate_script(&Gate { + conclusion: "failure", + results: Some(vec![("refused.txt", nightly.as_str())]), + ..Gate::default() + }); + assert_eq!(window.code, 0, "{}", window.log); + assert!(window.output.contains("skipped=true"), "{}", window.output); + assert!( + window.summary.contains("window has passed"), + "{}", + window.summary + ); + assert!(window.report.is_empty()); + // Any other refusal stays red, with the queue's reason, and is no skip. + // Only the first line counts: `nightly:` later in the file is not a window. + let capacity = run_gate_script(&Gate { + conclusion: "failure", + results: Some(vec![( + "refused.txt", + "capacity: too many runs\nnightly: not this line\n", + )]), + ..Gate::default() + }); + assert_eq!(capacity.code, 1, "{}", capacity.log); + assert!(!capacity.output.contains("skipped"), "{}", capacity.output); + assert!( + capacity.log.contains("capacity: too many runs"), + "{}", + capacity.log + ); + assert!( + capacity.summary.contains("capacity: too many runs"), + "{}", + capacity.summary + ); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_skips_for_a_missing_workflow_only_when_the_branch_adds_it() { + if !jq_available() { + return; + } + // Not on main, but on the branch: the pull request that adds it. + let adds = run_gate_script(&Gate { + main: "absent", + ..Gate::default() + }); + assert_eq!(adds.code, 0, "{}", adds.log); + assert!(adds.output.contains("skipped=true"), "{}", adds.output); + assert!( + adds.summary.contains("adds queue-pr-run.yml"), + "{}", + adds.summary + ); + assert_eq!( + adds.calls_starting("workflow run"), + 0, + "nothing is dispatched" + ); + // Gone from both (renamed or removed): red, never a silent green. + let gone = run_gate_script(&Gate { + main: "absent", + head: "absent", + ..Gate::default() + }); + assert_eq!(gone.code, 1, "{}", gone.log); + assert!(!gone.output.contains("skipped"), "{}", gone.output); + assert_eq!(gone.calls_starting("workflow run"), 0); + // An error that is not a 404 says nothing about the file: red. + let broken = run_gate_script(&Gate { + main: "error", + ..Gate::default() + }); + assert_eq!(broken.code, 1, "{}", broken.log); + assert!(!broken.output.contains("skipped"), "{}", broken.output); + assert!(broken.log.contains("HTTP 502"), "{}", broken.log); + assert_eq!(broken.calls_starting("workflow run"), 0); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_asks_again_after_a_failed_api_call() { + if !jq_available() { + return; + } + let run = run_gate_script(&Gate { + list_fail: 2, + status_fail: 1, + status_running: 2, + conclusion_fail: 2, + ..Gate::default() + }); + assert_eq!(run.code, 0, "{}", run.log); + // The list is asked again until it answers: 2 failures, then the answer. + assert_eq!(run.calls_starting("run list"), 3, "{:?}", run.calls); + // The poll rides out a failure and waits a minute a round: 1 + 2 + 1 calls. + assert_eq!( + run.calls_starting("run view 222 --json status"), + 4, + "{:?}", + run.calls + ); + assert_eq!(run.calls.iter().filter(|c| *c == "sleep 60").count(), 3); + // The verdict is asked again after a pause that grows. + assert_eq!( + run.calls_starting("run view 222 --json conclusion"), + 3, + "{:?}", + run.calls + ); + assert!( + run.calls.contains(&"sleep 10".to_owned()), + "{:?}", + run.calls + ); + assert!( + run.calls.contains(&"sleep 20".to_owned()), + "{:?}", + run.calls + ); + // A verdict that never comes is `unknown`, which is red, not green. + let never = run_gate_script(&Gate { + conclusion_fail: 99, + ..Gate::default() + }); + assert_eq!(never.code, 1, "{}", never.log); + assert_eq!(never.calls_starting("run view 222 --json conclusion"), 5); + assert!(never.report.contains(&"report.json".to_owned())); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_fails_when_the_dispatched_run_never_appears() { + if !jq_available() { + return; + } + let run = run_gate_script(&Gate { + listed: false, + ..Gate::default() + }); + assert_eq!(run.code, 1, "{}", run.log); + assert!(run.log.contains("did not appear"), "{}", run.log); + assert_eq!(run.calls_starting("run list"), 36, "{:?}", run.calls); + assert_eq!(run.calls_starting("run download"), 0); + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_refuses_a_branch_name_the_queue_does_not_take() { + if !jq_available() { + return; + } + for bad in ["-leading-dash", "has space", "tilde~1", "at@{1}", ""] { + let run = run_gate_script(&Gate { + head_ref: bad, + ..Gate::default() + }); + assert_eq!(run.code, 1, "{bad:?}: {}", run.log); + assert_eq!( + run.calls_starting("workflow run"), + 0, + "{bad:?}: nothing is dispatched" + ); + // It says which rule failed, and does not echo the author's text. + assert!(run.log.contains("at most 200 characters"), "{}", run.log); + if !bad.is_empty() { + assert!(!run.log.contains(bad), "{bad:?} is echoed: {}", run.log); + } + } + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_does_nothing_until_the_queue_side_switch_is_on() { + if !jq_available() { + return; + } + for switch in ["", "false", "TRUE", "1"] { + let run = run_gate_script(&Gate { + switch, + ..Gate::default() + }); + assert_eq!(run.code, 0, "{switch:?}: {}", run.log); + assert!( + run.output.contains("skipped=true"), + "{switch:?}: {}", + run.output + ); + assert!(run.summary.contains("QUEUE_PR_GATE"), "{}", run.summary); + assert!( + run.calls.is_empty(), + "{switch:?}: nothing is called: {:?}", + run.calls + ); + } + } + + #[test] + #[cfg(unix)] + fn queue_gate_script_gives_up_on_a_run_it_cannot_read() { + if !jq_available() { + return; + } + let run = run_gate_script(&Gate { + status_fail: 99, + ..Gate::default() + }); + assert_eq!(run.code, 1, "{}", run.log); + assert_eq!( + run.calls_starting("run view 222 --json status"), + 10, + "{:?}", + run.calls + ); + assert!(run.log.contains("10 times in a row"), "{}", run.log); + // Failures that are not in a row do not add up: 9, a success, 9 more is fine. + let spaced = run_gate_script(&Gate { + status_fail: 9, + ..Gate::default() + }); + assert_eq!(spaced.code, 0, "{}", spaced.log); + } + + #[test] + #[cfg(unix)] + fn queue_gate_cancel_step_finds_a_run_that_is_not_listed_yet() { + if !jq_available() { + return; + } + let cancel = |gate: Gate| { + run_gate_script(&Gate { + cancel_step: true, + ..gate + }) + }; + // Dispatched, id not found: asked again until the run is listed. + let late = cancel(Gate { + list_fail: 3, + ..Gate::default() + }); + assert_eq!(late.code, 0, "{}", late.log); + assert_eq!(late.calls_starting("run list"), 4, "{:?}", late.calls); + assert!( + late.calls.contains(&"run cancel 222".to_owned()), + "{:?}", + late.calls + ); + // Never listed: it stops after as many tries as the main step allows. + let never = cancel(Gate { + listed: false, + ..Gate::default() + }); + assert_eq!(never.code, 0, "{}", never.log); + assert_eq!(never.calls_starting("run list"), 36, "{:?}", never.calls); + assert_eq!(never.calls_starting("run cancel"), 0); + // Nothing dispatched: nothing to look for, and nothing is asked. + let none = cancel(Gate { + dispatched: false, + ..Gate::default() + }); + assert_eq!(none.code, 0, "{}", none.log); + assert!(none.calls.is_empty(), "{:?}", none.calls); + // The id is known: cancelled at once, without a lookup. + let known = cancel(Gate { + run_id_file: Some("333"), + ..Gate::default() + }); + assert_eq!(known.calls, ["run cancel 333"], "{:?}", known.calls); + } + + /// The gate's half of the queue contract is in the script and in the doc + /// that tells the queue side's owners what the gate reads: the mock runs above + /// use the same constants, so a changed prefix or path moves all four. + #[test] + fn queue_gate_contract_with_the_queue_side_is_in_the_script_and_the_doc() { + let text = read_workflow("e2e-selfhosted.yml"); + let gate = without_comments(job_block(&text, "e2e-gpu-queue")); + assert!( + gate.contains(&format!("== {QUEUE_NIGHTLY_REFUSAL}*")), + "the gate must read the nightly refusal by its `{QUEUE_NIGHTLY_REFUSAL}` prefix" + ); + assert!( + gate.contains("queue-results/refused.txt") + && gate.contains(&format!( + "results=queue-results/{}", + QUEUE_REPORT_PATH.trim_end_matches("/report.json") + )), + "the gate must read refused.txt and the MI300X release results directory" + ); + let doc = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../docs/nightly-on-the-queue.md"), + ) + .expect("read docs/nightly-on-the-queue.md"); + for needle in [QUEUE_NIGHTLY_REFUSAL, QUEUE_REPORT_PATH, "refused.txt"] { + assert!( + doc.contains(needle), + "docs/nightly-on-the-queue.md must state the queue contract: `{needle}`" + ); + } + } + #[test] fn queue_run_holds_no_credential() { assert_queue_workflow_holds_no_credential( @@ -9436,7 +11126,10 @@ permissions: { continue; } - let found = queue_runner_references(&read_workflow(&name), &workflow_name); + let mut found = queue_runner_references(&read_workflow(&name), &workflow_name); + if QUEUE_PR_RUN_CALLERS.contains(&name.as_ref()) { + found.retain(|word| word != "queue-pr-run"); + } assert!( found.is_empty(), "{name} names {found:?}: only {QUEUE_WORKFLOWS:?} may use the queue-report \ @@ -9477,7 +11170,7 @@ permissions: .is_some_and(word_char) }) }; - let mut found: Vec = ["queue-report", "nightly-queue", "queue-run"] + let mut found: Vec = ["queue-report", "nightly-queue", "queue-run", "queue-pr-run"] .into_iter() .filter(|word| names_word(word)) .map(str::to_owned) @@ -9517,6 +11210,10 @@ permissions: job(" uses: ./.github/workflows/queue-run.yml"), vec!["queue-run"], ), + ( + job(" run: gh workflow run queue-pr-run.yml --ref main"), + vec!["queue-pr-run"], + ), (job(" needs: [queue-run]"), vec!["queue-run"]), ( job(" needs:\n - queue-run\n - lint"), @@ -9568,6 +11265,9 @@ permissions: for (file, group) in [ ("queue-run.yml", "queue-run"), ("nightly-queue.yml", "nightly-queue"), + // Per branch, so pull requests' runs go side by side (the queue + // side limits how many); GitHub keeps one waiting per group. + ("queue-pr-run.yml", "queue-pr-run-${{ inputs.branch }}"), ] { let text = read_workflow(file); // A publisher that may fail without failing the job reports a @@ -9653,8 +11353,8 @@ permissions: let flat = doc.split_whitespace().collect::>().join(" "); for needed in [ "both the `release` and the `nightly` channel", - "The remaining gap is native Windows and WSL.", - "the native Windows and WSL lanes stay on their dedicated runners", + "The remaining gap is Strix Halo.", + "the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool", "`E2E_CHANNELS`", "`/out//results`", ] { @@ -9711,6 +11411,10 @@ permissions: ); let run_text = format!("{} h", run_secs / 3600); assert_eq!(run_secs % 3600, 0, "the run limit is whole hours"); + assert!( + flat.contains(&format!("{run_text} ({run_secs} s")), + "the doc's start section must give the run limit as {run_text} ({run_secs} s" + ); let text = read_workflow("queue-run.yml"); assert!( text.contains(&format!("{wait_secs} s ({wait_text})")), @@ -9731,9 +11435,23 @@ permissions: timeout_minutes * 60 >= wait_secs + run_secs + 30 * 60, "timeout-minutes: {timeout_minutes} leaves less than 30 minutes after a {wait_secs} s wait and a {run_secs} s run" ); + // What the timeout holds beyond the two limits and the 30 minutes: the + // margin the comment names. Derived, so it cannot drift from the number. + assert_eq!( + (wait_secs + run_secs) % 60, + 0, + "the limits are whole minutes" + ); + let margin_minutes = timeout_minutes - (wait_secs + run_secs) / 60 - 30; + // The comment's last lines and the timeout line, exactly: a stale total or + // margin anywhere else in the file cannot satisfy this. assert!( - text.contains(&format!(": {timeout_minutes} min.")), - "queue-run.yml's comment must give the timeout as {timeout_minutes} min" + text.contains(&format!( + " # publishing the run can take 30 min more, and {margin_minutes} min are margin for the\n\ + \x20 # runner's own delays: {timeout_minutes} min.\n\ + \x20 timeout-minutes: {timeout_minutes}\n" + )), + "queue-run.yml's comment must give the 30 min, the {margin_minutes} min margin and the {timeout_minutes} min total, right above timeout-minutes" ); } @@ -9743,6 +11461,7 @@ permissions: for (file, artifact) in [ ("queue-run.yml", "queue-run-results"), ("nightly-queue.yml", "nightly-queue-results"), + ("queue-pr-run.yml", "queue-pr-run-results"), ] { let upload = yaml_block(file, " - name: Upload lane results"); // Uploaded whatever the run's verdict: a failed run's files are